From 8d62394458ff823ee66b0d951f6f5d6ae2d08c4c Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 16 Mar 2018 10:13:17 -0400 Subject: [PATCH 1/7] Updated TestComm --- tests/CMakeLists.txt | 2 +- tests/TestCommD3Q19.cpp | 235 ++++++++++++++++++++-------------------- 2 files changed, 121 insertions(+), 116 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 86dece80..6ee340a8 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -52,7 +52,7 @@ ADD_LBPM_TEST_1_2_4( TestBlobIdentify ) #ADD_LBPM_TEST_PARALLEL( TestTwoPhase 8 ) ADD_LBPM_TEST_PARALLEL( TestBlobAnalyze 8 ) ADD_LBPM_TEST_PARALLEL( TestSegDist 8 ) -#ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 ) +ADD_LBPM_TEST_PARALLEL( TestCommD3Q19 8 ) #ADD_LBPM_TEST_PARALLEL( TestMassConservationD3Q7 1 ) ADD_LBPM_TEST_1_2_4( testCommunication ) ADD_LBPM_TEST_1_2_4( testUtilities ) diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index 068e7343..b9346413 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -11,7 +11,7 @@ using namespace std; -extern void GlobalFlipScaLBL_D3Q19_Init(double *dist_even, double *dist_odd, int Nx, int Ny, int Nz, +extern void GlobalFlipScaLBL_D3Q19_Init(double *dist, IntArray Map, int Np, int Nx, int Ny, int Nz, int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz) { // Set of Discrete velocities for the D3Q19 Model @@ -24,64 +24,57 @@ extern void GlobalFlipScaLBL_D3Q19_Init(double *dist_even, double *dist_odd, int int x,y,z; // Global indices int xn,yn,zn; // Global indices of neighbor int X,Y,Z; // Global size + int idx; X = Nx*nprocx; Y = Ny*nprocy; Z = Nz*nprocz; NULL_USE(Z); N = (Nx+2)*(Ny+2)*(Nz+2); // size of the array including halo + + for (k=0; k 0){ + // Get the 'global' index + x = iproc*Nx+i; + y = jproc*Ny+j; + z = kproc*Nz+k; + for (q=0; q<18; q++){ + // Odd distribution + Cqx = D3Q19[q][0]; + Cqy = D3Q19[q][1]; + Cqz = D3Q19[q][2]; + xn = x - Cqx; + yn = y - Cqy; + zn = z - Cqz; + xn=x; yn=y;zn=z; + if (xn < 0) xn += nprocx*Nx; + if (yn < 0) yn += nprocy*Ny; + if (zn < 0) zn += nprocz*Nz; + if (!(xn < nprocx*Nx)) xn -= nprocx*Nx; + if (!(yn < nprocy*Ny)) yn -= nprocy*Ny; + if (!(zn < nprocz*Nz)) zn -= nprocz*Nz; + + dist[(q+1)*Np+idx] = (zn*X*Y+yn*X+xn) + (q+1)*0.01; + + } } } } } - } -extern int GlobalCheckDebugDist(double *dist_even, double *dist_odd, int Nx, int Ny, int Nz, - int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz) +extern int GlobalCheckDebugDist(double *dist, IntArray Map, int Np, int Nx, int Ny, int Nz, + int iproc, int jproc, int kproc, int nprocx, int nprocy, int nprocz, int start, int finish) { int returnValue = 0; - int q,i,j,k,n,N; + int q,i,j,k,n,N,idx; int Cqx,Cqy,Cqz; // Discrete velocity int x,y,z; // Global indices int xn,yn,zn; // Global indices of neighbor @@ -89,37 +82,30 @@ extern int GlobalCheckDebugDist(double *dist_even, double *dist_odd, int Nx, int X = Nx*nprocx; Y = Ny*nprocy; Z = Nz*nprocz; - NULL_USE(Z); + NULL_USE(Z); N = (Nx+2)*(Ny+2)*(Nz+2); // size of the array including halo for (k=0; k start && idx< finish){ + // Get the 'global' index + x = iproc*Nx+i; + y = jproc*Ny+j; + z = kproc*Nz+k; + for (q=0; q<18; q++){ - if (dist_even[(q+1)*N+n] != (z*X*Y+y*X+x) + 2*(q+1)*0.01){ - printf("******************************************\n"); - printf("error in even distribution q = %i \n", 2*(q+1)); - printf("i,j,k= %i, %i, %i \n", x,y,z); - printf("dist = %5.2f \n", dist_even[(q+1)*N+n]); - printf("n= %i \n",z*X*Y+y*X+x); - returnValue++; - } + if (dist[(q+1)*Np+idx] != (z*X*Y+y*X+x) + (q+1)*0.01){ + printf("******************************************\n"); + printf("error in distribution q = %i \n", (q+1)); + printf("i,j,k= %i, %i, %i \n", x,y,z); + printf("dist = %5.2f \n", dist[(q+1)*Np+idx]); + printf("n= %i \n",z*X*Y+y*X+x); + returnValue++; + } - - if (dist_odd[q*N+n] != (z*X*Y+y*X+x) + (2*q+1)*0.01){ - printf("******************************************\n"); - printf("error in odd distribution q = %i \n", 2*q+1); - printf("i,j,k= %i, %i, %i \n", x,y,z); - printf("dist = %5.2f \n", dist_odd[q*N+n]); - printf("n= %i \n",z*X*Y+y*X+x); - returnValue++; } } } @@ -337,6 +323,7 @@ int main(int argc, char **argv) double sum; double sum_local=0.0, porosity; char component = 0; // solid phase + int Np=0; for (k=1;k Date: Fri, 16 Mar 2018 11:04:24 -0400 Subject: [PATCH 2/7] Read IDs in TestComm --- tests/TestCommD3Q19.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index b9346413..9934cd8d 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -297,14 +297,14 @@ int main(int argc, char **argv) char *id; id = new char[Nx*Ny*Nz]; - /* - * if (rank==0) printf("Assigning phase ID from file \n"); - * if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); - FILE *IDFILE = fopen(LocalRankFilename,"rb"); - if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); - fread(id,1,N,IDFILE); - fclose(IDFILE); - */ + + if (rank==0) printf("Assigning phase ID from file \n"); + if (rank==0) printf("Initialize from segmented data: solid=0, NWP=1, WP=2 \n"); + FILE *IDFILE = fopen(LocalRankFilename,"rb"); + if (IDFILE==NULL) ERROR("Error opening file: ID.xxxxx"); + fread(id,1,N,IDFILE); + fclose(IDFILE); + // Setup the domain for (k=0;k Date: Fri, 16 Mar 2018 11:27:17 -0400 Subject: [PATCH 3/7] TestComm to read IDs --- tests/TestCommD3Q19.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index 9934cd8d..6030cf82 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -310,7 +310,7 @@ int main(int argc, char **argv) for (j=0;j Date: Fri, 16 Mar 2018 12:23:12 -0400 Subject: [PATCH 4/7] Fixing perf measures in TestCommD3Q19 --- tests/TestCommD3Q19.cpp | 6 +++--- tests/lbpm_color_macro_simulator.cpp | 12 +----------- tests/lbpm_permeability_simulator.cpp | 2 +- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/tests/TestCommD3Q19.cpp b/tests/TestCommD3Q19.cpp index 9934cd8d..5d7dcaf6 100644 --- a/tests/TestCommD3Q19.cpp +++ b/tests/TestCommD3Q19.cpp @@ -481,7 +481,7 @@ int main(int argc, char **argv) // cout << "CPU time: " << (stoptime - starttime) << " seconds" << endl; cputime = stoptime - starttime; // cout << "Lattice update rate: "<< double(Nx*Ny*Nz*timestep)/cputime/1000000 << " MLUPS" << endl; - double MLUPS = double(Nx*Ny*Nz*timestep)/cputime/1000000; + double MLUPS = double(Np)*double(timestep)/cputime*1e-6; if (rank==0) printf("********************************************************\n"); if (rank==0) printf("CPU time = %f \n", cputime); if (rank==0) printf("Lattice update rate (per process)= %f MLUPS \n", MLUPS); @@ -491,9 +491,9 @@ int main(int argc, char **argv) // Number of memory references from the swap algorithm (per timestep) // 18 reads and 18 writes for each lattice site - double MemoryRefs = (Nx-2)*(Ny-2)*(Nz-2)*36; + double MemoryRefs = double(Np)*36; // number of memory references for the swap algorithm - GigaBytes / second - if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*8*timestep/1e9); + if (rank==0) printf("DRAM bandwidth (per process)= %f GB/sec \n",MemoryRefs*8*double(timestep)*1e-9); // Report bandwidth in Gigabits per second // communication bandwidth includes both send and recieve if (rank==0) printf("Communication bandwidth (per process)= %f Gbit/sec \n",ScaLBL_Comm.CommunicationCount*64*timestep/1e9); diff --git a/tests/lbpm_color_macro_simulator.cpp b/tests/lbpm_color_macro_simulator.cpp index dc388287..6107bab2 100644 --- a/tests/lbpm_color_macro_simulator.cpp +++ b/tests/lbpm_color_macro_simulator.cpp @@ -645,7 +645,7 @@ int main(int argc, char **argv) // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; // Performance obtained from each node - double MLUPS = double(Nx*Ny*Nz)/cputime/1000000; + double MLUPS = double(Np)/cputime/1000000; if (rank==0) printf("********************************************************\n"); if (rank==0) printf("CPU time = %f \n", cputime); @@ -655,16 +655,6 @@ int main(int argc, char **argv) if (rank==0) printf("********************************************************\n"); // ************************************************************************ - double *PHASE; - int SIZE=Nx*Ny*Nz*sizeof(double); - PHASE= new double [Nx*Ny*Nz]; - ScaLBL_CopyToHost(&PHASE[0],&Phi[0],SIZE); - - FILE *OUTFILE; - sprintf(LocalRankFilename,"Phase.%05i.raw",rank); - OUTFILE = fopen(LocalRankFilename,"wb"); - fwrite(PHASE,8,N,OUTFILE); - fclose(OUTFILE); PROFILE_STOP("Main"); PROFILE_SAVE("lbpm_color_simulator",1); diff --git a/tests/lbpm_permeability_simulator.cpp b/tests/lbpm_permeability_simulator.cpp index 723b1e4e..4238bb84 100644 --- a/tests/lbpm_permeability_simulator.cpp +++ b/tests/lbpm_permeability_simulator.cpp @@ -424,7 +424,7 @@ int main(int argc, char **argv) // Compute the walltime per timestep cputime = (stoptime - starttime)/timestep; // Performance obtained from each node - double MLUPS = double(Nx*Ny*Nz)/cputime/1000000; + double MLUPS = double(Np)/cputime/1000000; if (rank==0) printf("********************************************************\n"); if (rank==0) printf("CPU time = %f \n", cputime); From 782bb2b658ccabd333c725b96d0810f3a696a286 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 16 Mar 2018 12:24:32 -0400 Subject: [PATCH 5/7] Added cases for summit --- example/systems/summit/1200p/Color.in | 6 +++ example/systems/summit/1200p/Domain.in | 3 ++ example/systems/summit/3600p/Color.in | 6 +++ example/systems/summit/3600p/Domain.in | 3 ++ example/systems/summit/3600p/Permeability.in | 5 ++ example/systems/summit/Domain.in | 3 ++ .../systems/summit/GeneratePeriodicCase.sh | 13 +++++ example/systems/summit/summit-spheres-1g.lsf | 48 +++++++++++++++++++ example/systems/summit/summit-test-1200g.lsf | 26 ++++++++++ example/systems/summit/summit-test-3600g.lsf | 26 ++++++++++ 10 files changed, 139 insertions(+) create mode 100644 example/systems/summit/1200p/Color.in create mode 100644 example/systems/summit/1200p/Domain.in create mode 100644 example/systems/summit/3600p/Color.in create mode 100644 example/systems/summit/3600p/Domain.in create mode 100644 example/systems/summit/3600p/Permeability.in create mode 100644 example/systems/summit/Domain.in create mode 100755 example/systems/summit/GeneratePeriodicCase.sh create mode 100644 example/systems/summit/summit-spheres-1g.lsf create mode 100644 example/systems/summit/summit-test-1200g.lsf create mode 100644 example/systems/summit/summit-test-3600g.lsf diff --git a/example/systems/summit/1200p/Color.in b/example/systems/summit/1200p/Color.in new file mode 100644 index 00000000..32f16e0a --- /dev/null +++ b/example/systems/summit/1200p/Color.in @@ -0,0 +1,6 @@ +1.0 1.0 +1.0 1.0 +1.0e-3 0.95 +0.0 0.0 1.0e-6 +0 0 10.0 1.0 +100 5000 1e-5 diff --git a/example/systems/summit/1200p/Domain.in b/example/systems/summit/1200p/Domain.in new file mode 100644 index 00000000..17adf911 --- /dev/null +++ b/example/systems/summit/1200p/Domain.in @@ -0,0 +1,3 @@ +10 10 12 +320 320 320 +1.0 1.0 1.0 diff --git a/example/systems/summit/3600p/Color.in b/example/systems/summit/3600p/Color.in new file mode 100644 index 00000000..32f16e0a --- /dev/null +++ b/example/systems/summit/3600p/Color.in @@ -0,0 +1,6 @@ +1.0 1.0 +1.0 1.0 +1.0e-3 0.95 +0.0 0.0 1.0e-6 +0 0 10.0 1.0 +100 5000 1e-5 diff --git a/example/systems/summit/3600p/Domain.in b/example/systems/summit/3600p/Domain.in new file mode 100644 index 00000000..bca59233 --- /dev/null +++ b/example/systems/summit/3600p/Domain.in @@ -0,0 +1,3 @@ +15 15 16 +320 320 320 +1.0 1.0 1.0 diff --git a/example/systems/summit/3600p/Permeability.in b/example/systems/summit/3600p/Permeability.in new file mode 100644 index 00000000..8ee42b7b --- /dev/null +++ b/example/systems/summit/3600p/Permeability.in @@ -0,0 +1,5 @@ +1.0 +0.0 0.0 1.0e-6 +0 0 1.0 1.0 +100 1000 1.0e-5 + diff --git a/example/systems/summit/Domain.in b/example/systems/summit/Domain.in new file mode 100644 index 00000000..b509cc43 --- /dev/null +++ b/example/systems/summit/Domain.in @@ -0,0 +1,3 @@ +1 1 1 +320 320 320 +1.0 1.0 1.0 diff --git a/example/systems/summit/GeneratePeriodicCase.sh b/example/systems/summit/GeneratePeriodicCase.sh new file mode 100755 index 00000000..ddaeec47 --- /dev/null +++ b/example/systems/summit/GeneratePeriodicCase.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +NRANKS=3600 +echo $NRANKS + +DIR=$NRANKS"p" +mkdir -p $DIR + +BASEDIST="SignDist.0" +BASEID="ID.0" + +for i in `seq -w 0 $NRANKS`; do idfile="$BASEID$i"; echo $idfile; cp ID.00000 $DIR/$idfile; done +for i in `seq -w 0 $NRANKS`; do distfile="$BASEDIST$i"; echo $distfile; cp SignDist.00000 $DIR/$distfile; done diff --git a/example/systems/summit/summit-spheres-1g.lsf b/example/systems/summit/summit-spheres-1g.lsf new file mode 100644 index 00000000..23a9c71e --- /dev/null +++ b/example/systems/summit/summit-spheres-1g.lsf @@ -0,0 +1,48 @@ +#!/bin/bash +#BSUB -P CSC275MCCLURE +#BSUB -J spheres +#BSUB -o spheres.o%J +#BSUB -W 10 +#BSUB -nnodes 1 +##BSUB -env "all,JOB_FEATURE=NVME" + +date + +module load gcc cuda + +export SCALBL_DIR=$HOME/summit/build/LBPM-WIA/tests + +jsrun -n1 -r1 -g1 -c1 -brs $SCALBL_DIR/GenerateSphereTest 1896 + + + +# Create the 1200 GPU case +NRANKS=1200 +echo $NRANKS + +DIR=$NRANKS"p" +mkdir -p $DIR + +BASEDIST="SignDist.0" +BASEID="ID.0" + +for i in `seq -w 0 $NRANKS`; do idfile="$BASEID$i"; echo $idfile; cp ID.00000 $DIR/$idfile; done +for i in `seq -w 0 $NRANKS`; do distfile="$BASEDIST$i"; echo $distfile; cp SignDist.00000 $DIR/$distfile; done + + +# Create the 3600 GPU case +NRANKS=3600 +echo $NRANKS + +DIR=$NRANKS"p" +mkdir -p $DIR + +BASEDIST="SignDist.0" +BASEID="ID.0" + +for i in `seq -w 0 $NRANKS`; do idfile="$BASEID$i"; echo $idfile; cp ID.00000 $DIR/$idfile; done +for i in `seq -w 0 $NRANKS`; do distfile="$BASEDIST$i"; echo $distfile; cp SignDist.00000 $DIR/$distfile; done + + +exit; + diff --git a/example/systems/summit/summit-test-1200g.lsf b/example/systems/summit/summit-test-1200g.lsf new file mode 100644 index 00000000..89d553d5 --- /dev/null +++ b/example/systems/summit/summit-test-1200g.lsf @@ -0,0 +1,26 @@ +#!/bin/bash + +#BSUB -P CSC275MCCLURE +#BSUB -J COLOR +#BSUB -o test-1200g.o%J +#BSUB -W 10 +#BSUB -nnodes 200 +##BSUB -env "all,JOB_FEATURE=NVME" + +date + +module load gcc cuda +#source $OLCF_SPECTRUM_MPI_ROOT/jsm_pmix/bin/export_smpi_env -gpu + +#cd /ccs/home/mcclurej/summit/build/ScaLBL/example/Sph1896/ + +#cp Domain.in.8g Domain.in + +export LBPM_WIA_DIR=$HOME/summit/build/LBPM-WIA/tests + +cd /gpfs/alpinetds/csc275/scratch/mcclurej/SCALING/WEAK/1200p + +jsrun -n1200 -r6 -g1 -c1 -brs --smpiargs="-gpu" $LBPM_WIA_DIR/TestCommD3Q19 + + +exit; diff --git a/example/systems/summit/summit-test-3600g.lsf b/example/systems/summit/summit-test-3600g.lsf new file mode 100644 index 00000000..9d839bef --- /dev/null +++ b/example/systems/summit/summit-test-3600g.lsf @@ -0,0 +1,26 @@ +#!/bin/bash + +#BSUB -P CSC275MCCLURE +#BSUB -J COLOR +#BSUB -o color-3600g.o%J +#BSUB -W 10 +#BSUB -nnodes 600 +##BSUB -env "all,JOB_FEATURE=NVME" + +date + +module load gcc cuda +#source $OLCF_SPECTRUM_MPI_ROOT/jsm_pmix/bin/export_smpi_env -gpu + +#cd /ccs/home/mcclurej/summit/build/ScaLBL/example/Sph1896/ + +#cp Domain.in.8g Domain.in + +export LBPM_WIA_DIR=$HOME/summit/build/LBPM-WIA/tests + +cd /gpfs/alpinetds/csc275/scratch/mcclurej/SCALING/WEAK/3600p + +jsrun -n3600 -r6 -g1 -c1 -brs --smpiargs="-gpu" $LBPM_WIA_DIR/TestCommD3Q19 + + +exit; From dabd144c38bad256bf57edba24440c6f06a1eb57 Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 16 Mar 2018 12:26:07 -0400 Subject: [PATCH 6/7] Added example output files --- example/systems/summit/test-1200g.log | 247 ++++++++++++++++++++++++++ example/systems/summit/test-3600g.log | 142 +++++++++++++++ 2 files changed, 389 insertions(+) create mode 100644 example/systems/summit/test-1200g.log create mode 100644 example/systems/summit/test-3600g.log diff --git a/example/systems/summit/test-1200g.log b/example/systems/summit/test-1200g.log new file mode 100644 index 00000000..fb6eeb2a --- /dev/null +++ b/example/systems/summit/test-1200g.log @@ -0,0 +1,247 @@ +Fri Mar 16 11:58:54 EDT 2018 +******************************************************** +Running Unit Test for D3Q19 MPI Communication +******************************************************** +******************************************************** +Sub-domain size = 320 x 320 x 320 +Parallel domain size = 10 x 10 x 12 +******************************************************** +Assigning phase ID from file +Initialize from segmented data: solid=0, NWP=1, WP=2 +Media porosity = 0.359970 +Domain set. +Create ScaLBL_Communicator +Set up memory efficient layout +Allocating distributions +Setting up device map and neighbor list +Setting the distributions, size = : 11795503 +******************************************************** +No. of timesteps for timing: 100 +******************************************************** +CPU time = 1.354868 +Lattice update rate (per process)= -705.856629 MLUPS +Lattice update rate (process)= -847027.954450 MLUPS +******************************************************** +DRAM bandwidth (per process)= 943.718400 GB/sec +Communication bandwidth (per process)= 2.813606 Gbit/sec +Aggregated communication bandwidth = 3376.327680 Gbit/sec +****************************************** +error in distribution q = 14 +i,j,k= 1281, 486, 2097 +dist = 1.14 +n= 1 + +------------------------------------------------------------ +Sender: LSF System +Subject: Job 43312: in cluster Exited + +Job was submitted from host by user in cluster at Fri Mar 16 11:58:33 2018 +Job was executed on host(s) <1*batch1>, in queue , as user in cluster at Fri Mar 16 11:58:35 2018 + <42*a20n15> + <42*a20n16> + <42*a20n17> + <42*a20n18> + <42*a21n01> + <42*a21n02> + <42*a21n03> + <42*a21n04> + <42*a28n07> + <42*a28n08> + <42*a28n09> + <42*a28n10> + <42*a28n11> + <42*a28n12> + <42*a28n13> + <42*a28n14> + <42*a28n15> + <42*a28n16> + <42*a28n17> + <42*a28n18> + <42*a29n01> + <42*a29n02> + <42*a29n03> + <42*a29n04> + <42*a29n05> + <42*a29n06> + <42*a29n07> + <42*a29n08> + <42*a29n09> + <42*a29n10> + <42*a29n11> + <42*a29n12> + <42*a29n13> + <42*a29n15> + <42*a29n16> + <42*a29n17> + <42*a29n18> + <42*a30n01> + <42*a30n02> + <42*a30n03> + <42*a30n04> + <42*a30n05> + <42*a30n06> + <42*a30n07> + <42*a30n08> + <42*a30n09> + <42*a30n10> + <42*a30n11> + <42*a30n12> + <42*a30n13> + <42*a30n14> + <42*a30n15> + <42*a30n16> + <42*a30n17> + <42*a30n18> + <42*a31n01> + <42*a31n02> + <42*a31n03> + <42*a31n04> + <42*a31n05> + <42*a31n06> + <42*a31n07> + <42*a31n08> + <42*a31n09> + <42*a31n10> + <42*a31n11> + <42*a31n12> + <42*a31n13> + <42*a31n14> + <42*a31n15> + <42*a31n16> + <42*a31n17> + <42*a31n18> + <42*a32n01> + <42*a32n02> + <42*a32n03> + <42*a32n04> + <42*a32n05> + <42*a32n06> + <42*a32n07> + <42*a32n08> + <42*a32n09> + <42*a32n10> + <42*a32n11> + <42*a32n12> + <42*a32n13> + <42*a32n14> + <42*a32n15> + <42*a32n16> + <42*a32n17> + <42*a32n18> + <42*a33n01> + <42*a33n02> + <42*a33n03> + <42*a33n04> + <42*a33n05> + <42*a33n06> + <42*a33n07> + <42*a33n08> + <42*a33n09> + <42*a33n10> + <42*a33n11> + <42*a33n12> + <42*a33n13> + <42*a33n14> + <42*a33n15> + <42*a33n16> + <42*a33n17> + <42*a33n18> + <42*a34n01> + <42*a34n02> + <42*a34n03> + <42*a34n05> + <42*a34n06> + <42*a34n07> + <42*a34n08> + <42*a34n09> + <42*a34n10> + <42*a34n11> + <42*a34n12> + <42*a34n13> + <42*a34n14> + <42*a34n16> + <42*a34n17> + <42*a34n18> + <42*a35n01> + <42*a35n02> + <42*a35n03> + <42*a35n04> + <42*a35n05> + <42*a35n06> + <42*a35n07> + <42*a35n08> + <42*a35n09> + <42*a35n10> + <42*a35n11> + <42*a35n12> + <42*a35n13> + <42*a35n14> + <42*a35n15> + <42*a35n16> + <42*a35n17> + <42*a35n18> + <42*a36n01> + <42*a36n02> + <42*a36n03> + <42*a36n04> + <42*a36n05> + <42*a36n06> + <42*a36n07> + <42*a36n09> + <42*a36n10> + <42*a36n11> + <42*a36n12> + <42*a36n13> + <42*a36n14> + <42*a36n15> + <42*a36n17> + <42*a36n18> + <42*b01n01> + <42*b01n02> + <42*b01n03> + <42*b01n04> + <42*b01n05> + <42*b01n06> + <42*b01n07> + <42*b01n08> + <42*b01n09> + <42*b01n10> + <42*b01n11> + <42*b01n12> + <42*b01n13> + <42*b01n14> + <42*b01n15> + <42*b01n16> + <42*b01n17> + <42*b01n18> + <42*b02n01> + <42*b02n02> + <42*b02n03> + <42*b02n04> + <42*b02n05> + <42*b02n06> + <42*b02n07> + <42*b02n09> + <42*b02n10> + <42*b02n11> + <42*b02n12> + <42*b02n13> + <42*b02n14> + <42*b02n15> + <42*b02n16> + <42*b02n17> + <42*b02n18> + <42*b03n01> + <42*b03n02> + <42*b03n03> + <42*b03n04> + <42*b03n05> + <42*b03n06> + was used as the home directory. + was used as the working directory. +Started at Fri Mar 16 11:58:35 2018 +Terminated at Fri Mar 16 11:59:26 2018 +Results reported at Fri Mar 16 11:59:26 2018 + +The output (if any) is above this job summary. + diff --git a/example/systems/summit/test-3600g.log b/example/systems/summit/test-3600g.log new file mode 100644 index 00000000..7a6f40a5 --- /dev/null +++ b/example/systems/summit/test-3600g.log @@ -0,0 +1,142 @@ +Fri Mar 16 11:29:36 EDT 2018 +******************************************************** +Running Unit Test for D3Q19 MPI Communication +******************************************************** +******************************************************** +Sub-domain size = 320 x 320 x 320 +Parallel domain size = 10 x 10 x 6 +******************************************************** +Assigning phase ID from file +Initialize from segmented data: solid=0, NWP=1, WP=2 +Media porosity = 0.359970 +Domain set. +Create ScaLBL_Communicator +Set up memory efficient layout +Allocating distributions +Setting up device map and neighbor list +Setting the distributions, size = : 11795503 +******************************************************** +No. of timesteps for timing: 100 +******************************************************** +CPU time = 1.329585 +Lattice update rate (per process)= -719.278687 MLUPS +Lattice update rate (process)= -431567.212128 MLUPS +******************************************************** +DRAM bandwidth (per process)= 943.718400 GB/sec +Communication bandwidth (per process)= 2.813606 Gbit/sec +Aggregated communication bandwidth = 1688.163840 Gbit/sec + +------------------------------------------------------------ +Sender: LSF System +Subject: Job 43296: in cluster Done + +Job was submitted from host by user in cluster at Fri Mar 16 11:28:16 2018 +Job was executed on host(s) <1*batch3>, in queue , as user in cluster at Fri Mar 16 11:29:16 2018 + <42*a07n04> + <42*a07n05> + <42*a07n10> + <42*a07n11> + <42*a07n12> + <42*a07n13> + <42*a07n14> + <42*a07n15> + <42*a07n16> + <42*a07n17> + <42*a08n01> + <42*a08n02> + <42*a08n03> + <42*a08n04> + <42*a08n05> + <42*a08n06> + <42*a08n07> + <42*a08n08> + <42*a08n09> + <42*a08n10> + <42*a08n11> + <42*a08n12> + <42*a08n14> + <42*a08n15> + <42*a08n16> + <42*a08n17> + <42*a08n18> + <42*a09n02> + <42*a09n03> + <42*a09n04> + <42*a09n05> + <42*a09n06> + <42*a09n07> + <42*a09n08> + <42*a09n09> + <42*a09n10> + <42*a09n11> + <42*a09n13> + <42*a09n14> + <42*a09n15> + <42*a09n16> + <42*a09n17> + <42*a09n18> + <42*a10n01> + <42*a10n02> + <42*a10n03> + <42*a10n04> + <42*a10n05> + <42*a10n06> + <42*a10n07> + <42*a10n08> + <42*a10n09> + <42*a10n10> + <42*a10n11> + <42*a10n12> + <42*a10n13> + <42*a10n14> + <42*a10n15> + <42*a10n16> + <42*a10n17> + <42*a10n18> + <42*a11n01> + <42*a11n02> + <42*a11n03> + <42*a11n04> + <42*a11n05> + <42*a15n02> + <42*a15n03> + <42*a15n04> + <42*a15n05> + <42*a15n06> + <42*a15n07> + <42*a15n08> + <42*a15n09> + <42*a15n10> + <42*a15n11> + <42*a15n12> + <42*a15n13> + <42*a15n14> + <42*a15n15> + <42*a15n16> + <42*a15n17> + <42*a15n18> + <42*a16n01> + <42*a16n02> + <42*a16n03> + <42*a16n04> + <42*a16n05> + <42*a16n06> + <42*a16n07> + <42*a16n08> + <42*a16n09> + <42*a16n10> + <42*a16n11> + <42*a16n12> + <42*a16n13> + <42*a16n14> + <42*a16n15> + <42*a16n16> + <42*a16n17> + was used as the home directory. + was used as the working directory. +Started at Fri Mar 16 11:29:16 2018 +Terminated at Fri Mar 16 11:29:59 2018 +Results reported at Fri Mar 16 11:29:59 2018 + +The output (if any) is above this job summary. + From e1d14cbac74727d01546e1b0b67f8f69b841747d Mon Sep 17 00:00:00 2001 From: James E McClure Date: Fri, 16 Mar 2018 13:00:42 -0400 Subject: [PATCH 7/7] Summit MPI testing scripts added --- example/systems/summit/pack.out | 1901 + example/systems/summit/test-1200g.log | 120 +- example/systems/summit/test-3600g.log | 61100 +++++++++++++++++++++++- 3 files changed, 62937 insertions(+), 184 deletions(-) create mode 100644 example/systems/summit/pack.out diff --git a/example/systems/summit/pack.out b/example/systems/summit/pack.out new file mode 100644 index 00000000..648ecb91 --- /dev/null +++ b/example/systems/summit/pack.out @@ -0,0 +1,1901 @@ +Number of Spheres: 1896 +Domain Length (x,y,z): 1, 1, 1 +Media porosity: 0.36 +log(r) Normal with mean -3.21402, variance 0.1 +Mean coordination No. 10.7773 +0.00783666 0.485601 0.710132 0.0569629 +0.8056 0.819831 0.789826 0.0411985 +0.107974 0.200585 0.562806 0.0604344 +0.597331 0.848588 0.670386 0.0318123 +0.328284 0.844009 0.385849 0.0416506 +0.622311 0.419675 0.157469 0.0486098 +0.499979 0.724703 0.432715 0.0418867 +0.517903 0.0790305 0.146302 0.0468057 +0.81903 0.386921 0.348704 0.039168 +0.272026 0.955045 0.64744 0.04012 +0.309342 0.328216 0.113999 0.042229 +0.344827 0.285321 0.727887 0.0547245 +0.721448 0.97397 0.81746 0.0413811 +0.11164 0.694361 0.434229 0.0750088 +0.532392 0.983456 0.444982 0.038981 +0.198212 0.237408 0.247742 0.0402747 +0.470635 0.194979 0.604732 0.052485 +0.779088 0.884067 0.126388 0.0317569 +0.29754 0.181501 0.332878 0.028369 +0.558505 0.290145 0.0306601 0.0359976 +0.996489 0.454515 0.321203 0.03095 +0.606997 0.687865 0.620731 0.0344868 +0.8883 0.315668 0.831943 0.0588586 +0.690178 0.234314 0.845459 0.0410889 +0.207609 0.161252 0.967885 0.0365915 +0.347991 0.0942915 0.5232 0.0632109 +0.97735 0.322714 0.44168 0.0556003 +0.893262 0.890354 0.534299 0.0418545 +0.127243 0.330682 0.161774 0.0419363 +0.112799 0.575721 0.99265 0.0364405 +0.90494 0.0889378 0.543044 0.0388716 +0.896255 0.0897912 0.860836 0.027202 +0.934843 0.45676 0.89771 0.0305272 +0.645066 0.212566 0.54365 0.0334649 +0.305301 0.394394 0.0716615 0.0364368 +0.894663 0.491618 0.859722 0.0348371 +0.301454 0.701154 0.557436 0.0349297 +0.356086 0.827433 0.572507 0.0463913 +0.0396797 0.129484 0.537471 0.0347806 +0.139703 0.543148 0.701646 0.0436696 +0.390023 0.0628284 0.247787 0.0271184 +0.442789 0.252093 0.0116819 0.0471642 +0.517471 0.66116 0.935257 0.0234323 +0.42346 0.574056 0.408563 0.041552 +0.516121 0.224238 0.915051 0.0685075 +0.36074 0.400451 0.697543 0.0653957 +0.449561 0.543052 0.217023 0.0535265 +0.754718 0.156662 0.249405 0.0320637 +0.448089 0.317162 0.194797 0.0263009 +0.673667 0.853829 0.905764 0.0780219 +0.840827 0.61118 0.136056 0.042174 +0.206534 0.00628898 0.141621 0.045277 +0.24475 0.389853 0.646931 0.0598005 +0.527627 0.162191 0.188388 0.0469041 +0.636439 0.371014 0.236598 0.038496 +0.686469 0.441908 0.517578 0.0315218 +0.520144 0.47856 0.262691 0.0524291 +0.00881108 0.800926 0.075799 0.0357783 +0.238457 0.351872 0.0614687 0.0434394 +0.0707515 0.460509 0.0166178 0.0299879 +0.939817 0.817021 0.669859 0.0249011 +0.44571 0.909412 0.136822 0.037056 +0.66254 0.448291 0.0584423 0.0468157 +0.0903611 0.822176 0.859917 0.0682131 +0.630596 0.43294 0.31722 0.0376087 +0.155128 0.656952 0.730243 0.0746815 +0.100771 0.134704 0.843696 0.0381953 +0.427078 0.133923 0.187261 0.0537489 +0.777194 0.145172 0.582113 0.0489597 +0.610205 0.927688 0.827324 0.0470197 +0.885134 0.188299 0.596336 0.0681439 +0.87759 0.29038 0.746248 0.0307454 +0.684208 0.754825 0.547477 0.027806 +0.832972 0.877714 0.691263 0.0396605 +0.793929 0.768617 0.559829 0.0274558 +0.589374 0.963482 0.555865 0.0268648 +0.734492 0.26669 0.682843 0.0278686 +0.639259 0.260462 0.681675 0.058492 +0.135111 0.0466873 0.623351 0.0301828 +0.162768 0.501143 0.468313 0.0330766 +0.831576 0.582202 0.246135 0.0586225 +0.803586 0.0319126 0.872713 0.0272617 +0.401939 0.106638 0.665729 0.0225154 +0.0154399 0.767054 0.739801 0.0472356 +0.624654 0.884919 0.188677 0.0237472 +0.745835 0.398506 0.0753145 0.0469003 +0.446789 0.998232 0.845681 0.0325518 +0.763375 0.584045 0.42691 0.0536573 +0.793642 0.0662929 0.0709492 0.0424104 +0.474591 0.604961 0.0240936 0.0307525 +0.20637 0.769027 0.275904 0.0642978 +0.794228 0.011128 0.453723 0.0481392 +0.691513 0.303075 0.106408 0.0584299 +0.465281 0.212966 0.501201 0.0429758 +0.473693 0.757029 0.94464 0.0416723 +0.0793349 0.0822443 0.795369 0.0356011 +0.0879652 0.339143 0.0621809 0.028057 +0.0892081 0.103562 0.651434 0.0481144 +0.769885 0.151599 0.0971989 0.0357893 +0.0319209 0.344468 0.831231 0.0432499 +0.685524 0.00707301 0.963266 0.0287685 +0.223584 0.722319 0.559564 0.041176 +0.834864 0.138375 0.172296 0.0365032 +0.745885 0.0754194 0.602373 0.0301365 +0.378988 0.106593 0.81684 0.051773 +0.5283 0.491028 0.147577 0.0636452 +0.436771 0.050761 0.934634 0.026795 +0.487009 0.542875 0.279912 0.0192804 +0.0509262 0.963117 0.62774 0.059891 +0.109226 0.97847 0.00297037 0.0351999 +0.625995 0.107322 0.374812 0.0277476 +0.208196 0.949463 0.963909 0.0350063 +0.0315404 0.798681 0.277237 0.0419151 +0.89333 0.442954 0.688839 0.0222062 +0.227076 0.33434 0.44306 0.0333213 +0.29497 0.977789 0.496102 0.0292425 +0.79618 0.872593 0.747258 0.0272462 +0.249905 0.478973 0.342757 0.0579416 +0.641968 0.346385 0.728466 0.0393086 +0.899765 0.828354 0.740461 0.0568704 +0.272218 0.728065 0.34209 0.031776 +0.838548 0.0231723 0.539975 0.0304454 +0.18998 0.184147 0.301252 0.0356707 +0.66046 0.0957828 0.866872 0.0375997 +0.742363 0.659691 0.716033 0.0280424 +0.0804224 0.336661 0.39084 0.044893 +0.450062 0.126479 0.535608 0.0429515 +0.168925 0.582837 0.829627 0.0500601 +0.216883 0.851716 0.0903591 0.0383953 +0.497409 0.106686 0.967526 0.0615782 +0.371425 0.027189 0.672969 0.0223365 +0.286199 0.275979 0.852738 0.0505746 +0.666184 0.160152 0.207047 0.061291 +0.975214 0.456821 0.485621 0.0485666 +0.354046 0.964105 0.929215 0.0269399 +0.652861 0.821683 0.55931 0.0457397 +0.663286 0.961664 0.288148 0.0586649 +0.780092 0.607092 0.340685 0.0371471 +0.415489 0.220381 0.44703 0.0302709 +0.331976 0.572517 0.870631 0.0336721 +0.85534 0.120975 0.405365 0.0330022 +0.898743 0.402806 0.408107 0.061506 +0.652296 0.0271124 0.892742 0.0362347 +0.00109096 0.649241 0.88736 0.0495611 +0.153486 0.85319 0.317667 0.0304793 +0.74619 0.976828 0.0748496 0.0420883 +0.462856 0.582151 0.944885 0.0366515 +0.554959 0.689043 0.718214 0.0453004 +0.748132 0.466069 0.117005 0.0325238 +0.437479 0.670469 0.977565 0.0577523 +0.16507 0.64997 0.946655 0.0339323 +0.316206 0.308884 0.330157 0.0607117 +0.0850523 0.771515 0.508251 0.0342484 +0.912787 0.488489 0.444478 0.0326296 +0.527657 0.378239 0.883133 0.0515176 +0.284376 0.226611 0.795117 0.0253242 +0.824591 0.453028 0.146049 0.0502992 +0.567799 0.131181 0.377221 0.0349595 +0.448415 0.985242 0.45695 0.0458634 +0.189904 0.750138 0.477809 0.0299629 +0.836012 0.0761238 0.832337 0.0408288 +0.926334 0.00476501 0.910503 0.0369942 +0.440438 0.606595 0.74895 0.0334315 +0.810899 0.32018 0.435827 0.0622357 +0.0636593 0.489291 0.0918926 0.0289976 +0.0896862 0.685575 0.315409 0.0341971 +0.66165 0.389439 0.449599 0.0513732 +0.234354 0.436745 0.153462 0.0537524 +0.509656 0.57437 0.755757 0.0432228 +0.10132 0.988747 0.457296 0.0483041 +0.583183 0.748462 0.668593 0.0370983 +0.122474 0.796274 0.0274527 0.0308615 +0.324982 0.00117287 0.761551 0.0327184 +0.813658 0.993577 0.97337 0.0809183 +0.202564 0.473538 0.608308 0.0387605 +0.92286 0.0563075 0.730636 0.0380546 +0.83953 0.874033 0.481091 0.0315262 +0.757419 0.686344 0.437332 0.0493433 +0.775039 0.138755 0.00537384 0.057073 +0.501141 0.319415 0.931617 0.0291315 +0.0827454 0.660513 0.908464 0.0339338 +0.6091 0.607484 0.674198 0.062034 +0.441995 0.195378 0.846132 0.0367416 +0.520239 0.69515 0.310851 0.0353533 +0.199418 0.72457 0.642135 0.0448879 +0.623463 0.737203 0.568869 0.0389638 +0.0410867 0.652444 0.968052 0.0376408 +0.638951 0.553418 0.858415 0.0522345 +0.838553 0.812034 0.970127 0.0370024 +0.543384 0.268348 0.431399 0.0462331 +0.940655 0.999154 0.782406 0.0410872 +0.017358 0.245895 0.961237 0.0458302 +0.248985 0.715591 0.491067 0.0294659 +0.857047 0.784034 0.240285 0.046548 +0.0615992 0.622065 0.859864 0.0220243 +0.604201 0.575195 0.562295 0.0167475 +0.665072 0.950641 0.458945 0.0279933 +0.6812 0.543957 0.714897 0.042323 +0.533576 0.910019 0.918191 0.0731504 +0.572516 0.825818 0.859079 0.0368503 +0.0765606 0.863733 0.306944 0.0425899 +0.67971 0.558897 0.457782 0.0249217 +0.915222 0.949448 0.49092 0.0247195 +0.0228084 0.00662286 0.0860634 0.0339621 +0.273411 0.290518 0.515971 0.0307921 +0.642623 0.772441 0.185435 0.0295154 +0.94172 0.628256 0.394925 0.0256003 +0.240219 0.933473 0.183526 0.0443671 +0.749072 0.627613 0.175557 0.0590654 +0.630776 0.177184 0.887657 0.0515026 +0.470924 0.874411 0.255435 0.0846582 +0.526935 0.716097 0.897549 0.04035 +0.467133 0.656264 0.878861 0.0462842 +0.495444 0.0772431 0.800527 0.0442704 +0.0181693 0.318043 0.376182 0.0217174 +0.206056 0.74285 0.712948 0.0222824 +0.593597 0.493042 0.305685 0.033905 +0.360211 0.309231 0.203378 0.0623501 +0.530204 0.663143 0.65657 0.0259377 +0.60661 0.815782 0.614478 0.0264928 +0.436127 0.398466 0.84905 0.0323365 +0.321115 0.0183708 0.304246 0.0724431 +0.731057 0.685216 0.680657 0.0170227 +0.167274 0.43129 0.90178 0.0361925 +0.946254 0.987823 0.460383 0.0333158 +0.922377 0.318741 0.228199 0.0365471 +0.741065 0.94137 0.153638 0.0358162 +0.648751 0.881972 0.715268 0.0441674 +0.811061 0.647441 0.642727 0.0334963 +0.20906 0.671289 0.193329 0.0479147 +0.825213 0.0217735 0.193165 0.0342723 +0.619392 0.0982672 0.0750044 0.0349937 +0.677736 0.541509 0.555423 0.0490032 +0.213625 0.488434 0.895686 0.0376384 +0.808744 0.831259 0.0411305 0.0386733 +0.833007 0.546011 0.00994336 0.0361181 +0.228785 0.628843 0.614483 0.0528521 +0.402583 0.00971101 0.981445 0.0442304 +0.727073 0.0635758 0.260393 0.0647258 +0.293552 0.430124 0.394137 0.024579 +0.782832 0.968364 0.205276 0.0349768 +0.423017 0.601311 0.05062 0.0273426 +0.625769 0.635435 0.186396 0.0637277 +0.95252 0.666023 0.0438117 0.0443656 +0.786723 0.55179 0.11565 0.0379311 +0.0186039 0.372613 0.149242 0.0269084 +0.513703 0.140251 0.469073 0.0501068 +0.91933 0.150455 0.698254 0.0374882 +0.105633 0.581414 0.356766 0.0620806 +0.370669 0.159302 0.618775 0.0545817 +0.631021 0.151863 0.660871 0.0385373 +0.496615 0.827354 0.673851 0.0456435 +0.239908 0.108883 0.355056 0.049866 +0.41217 0.673415 0.172341 0.022583 +0.791649 0.170941 0.198514 0.0222108 +0.0696419 0.471359 0.561036 0.0444965 +0.779623 0.308832 0.0265585 0.0606179 +0.257654 0.89666 0.861274 0.0360596 +0.00735261 0.163268 0.618201 0.0585136 +0.56229 0.795847 0.54129 0.0501532 +0.123058 0.468973 0.148728 0.0265708 +0.479672 0.368724 0.0219956 0.075594 +0.843557 0.846354 0.303238 0.0396086 +0.30011 0.0694759 0.102352 0.0742681 +0.490645 0.097259 0.207094 0.0222666 +0.0401963 0.645032 0.333142 0.0321915 +0.321136 0.840072 0.875918 0.0502333 +0.459964 0.812242 0.574348 0.0586082 +0.65355 0.278098 0.781941 0.0443111 +0.0570332 0.32273 0.197433 0.02295 +0.125103 0.711898 0.566274 0.058082 +0.130385 0.406795 0.616091 0.0598552 +0.994881 0.696791 0.800742 0.0480186 +0.0833753 0.270984 0.805445 0.0395709 +0.808304 0.222255 0.0807303 0.0454613 +0.0621235 0.591235 0.448956 0.0403329 +0.898918 0.289067 0.420053 0.0324385 +0.343677 0.61186 0.0190862 0.0444255 +0.0745187 0.94047 0.723294 0.0411039 +0.63399 0.98278 0.0938013 0.0294109 +0.814597 0.256135 0.784131 0.0417802 +0.0754806 0.856201 0.414838 0.0535544 +0.109962 0.434688 0.448373 0.0541147 +0.100779 0.897743 0.526971 0.0658756 +0.568284 0.231497 0.764552 0.054402 +0.623726 0.92524 0.151093 0.031382 +0.694635 0.538606 0.637642 0.0349845 +0.461722 0.641695 0.208489 0.0462293 +0.352881 0.905216 0.583001 0.0321617 +0.494538 0.859229 0.436385 0.0898921 +0.361805 0.669155 0.242813 0.0629294 +0.379218 0.794944 0.901772 0.0277315 +0.260445 0.278428 0.941207 0.041599 +0.34655 0.419305 0.439756 0.0461809 +0.560605 0.783418 0.12009 0.0232063 +0.668991 0.59104 0.495847 0.026039 +0.530669 0.5673 0.251003 0.0376935 +0.104009 0.121493 0.504214 0.0380757 +0.00896458 0.23089 0.830411 0.0414892 +0.624424 0.327084 0.521178 0.0505347 +0.7403 0.582317 0.60503 0.0358528 +0.232192 0.520327 0.183157 0.0349745 +0.863195 0.241764 0.181701 0.0711097 +0.882282 0.299867 0.0135257 0.0432533 +0.560756 0.420248 0.460795 0.0295846 +0.672377 0.0160602 0.641506 0.072129 +0.409699 0.900126 0.532497 0.0440284 +0.146362 0.28178 0.322676 0.061079 +0.956302 0.278414 0.186666 0.0260891 +0.0750988 0.0410901 0.62132 0.0219955 +0.840465 0.664536 0.842025 0.0632943 +0.167655 0.929474 0.0313139 0.0461509 +0.059528 0.272741 0.0236486 0.0341338 +0.801144 0.699558 0.582545 0.045601 +0.601414 0.0592492 0.519063 0.0446445 +0.819158 0.876137 0.206872 0.058499 +0.59724 0.381327 0.609889 0.0285596 +0.659337 0.620071 0.563979 0.0321365 +0.626787 0.800958 0.699226 0.0311793 +0.0358319 0.502841 0.303784 0.0312068 +0.491756 0.281518 0.567744 0.043968 +0.807121 0.835325 0.89969 0.0396875 +0.911597 0.628384 0.604031 0.0576898 +0.203794 0.252151 0.53693 0.0514143 +0.0490822 0.396374 0.754566 0.0509116 +0.193131 0.808782 0.498878 0.0271011 +0.115054 0.861987 0.716353 0.0475018 +0.802087 0.556899 0.615693 0.0318011 +0.859433 0.645307 0.0535112 0.0490647 +0.956176 0.674672 0.364683 0.0306942 +0.331724 0.613144 0.746679 0.0267933 +0.792235 0.307042 0.124111 0.0356845 +0.638922 0.456476 0.587278 0.0497489 +0.470229 0.0604778 0.629549 0.0675032 +0.0827105 0.100084 0.568459 0.0256152 +0.682957 0.149306 0.313731 0.0454596 +0.725544 0.37649 0.403578 0.0284271 +0.71824 0.325563 0.718062 0.0371119 +0.427004 0.450789 0.105899 0.0530468 +0.295924 0.523436 0.909544 0.0385973 +0.853306 0.101911 0.714644 0.0369238 +0.326275 0.136853 0.426327 0.0448055 +0.21039 0.370786 0.545064 0.0493774 +0.994769 0.346538 0.213289 0.0424189 +0.943031 0.0858459 0.659686 0.0414016 +0.532314 0.356092 0.604896 0.0375703 +0.283675 0.361361 0.413095 0.0340161 +0.690249 0.347551 0.871493 0.0748111 +0.0143496 0.41772 0.39742 0.0555479 +0.270434 0.638692 0.377443 0.0378579 +0.612662 0.317078 0.386409 0.0337887 +0.522555 0.7239 0.605225 0.053976 +0.901491 0.397469 0.50626 0.0268041 +0.226592 0.931242 0.755676 0.0792953 +0.79379 0.0879897 0.435019 0.0309667 +0.444921 0.509291 0.320292 0.0471743 +0.596221 0.43608 0.23843 0.038019 +0.527348 0.39341 0.430895 0.0213717 +0.585488 0.00325004 0.888589 0.0348285 +0.309374 0.249828 0.247751 0.0275504 +0.313725 0.775007 0.44518 0.0505094 +0.140262 0.321526 0.476282 0.0605114 +0.317928 0.863187 0.499485 0.0433927 +0.148138 0.179802 0.232188 0.0376209 +0.785895 0.232463 0.306224 0.0373758 +0.594102 0.889867 0.528735 0.0493206 +0.903989 0.128505 0.787715 0.0558957 +0.834798 0.394788 0.0749916 0.0421415 +0.14196 0.639751 0.622684 0.0350379 +0.400152 0.830318 0.51032 0.0298373 +0.732423 0.435823 0.676741 0.0812955 +0.938775 0.621974 0.848436 0.0288059 +0.757164 0.399586 0.482714 0.0407906 +0.383191 0.474119 0.212552 0.0422689 +0.99998 0.703688 0.442064 0.0313199 +0.809909 0.0886226 0.209917 0.0306789 +0.676291 0.036224 0.0451658 0.0543215 +0.959773 0.384112 0.793529 0.0472952 +0.191898 0.639529 0.363286 0.0403968 +0.101949 0.226064 0.747646 0.0359504 +0.463741 0.247947 0.402596 0.0408806 +0.803364 0.744386 0.697646 0.0779408 +0.427856 0.757771 0.209547 0.0478763 +0.276816 0.510562 0.51625 0.0515835 +0.36653 0.223487 0.811001 0.0511175 +0.0664635 0.854684 0.21497 0.0488211 +0.949236 0.230857 0.263781 0.0452797 +0.0254624 0.157661 0.991536 0.0416001 +0.012159 0.337893 0.0185734 0.0465778 +0.174397 0.495799 0.163057 0.0309484 +0.336058 0.0935292 0.714153 0.060291 +0.734087 0.25425 0.753876 0.0442475 +0.768961 0.806333 0.273381 0.0501561 +0.477749 0.153842 0.367608 0.0583945 +0.495394 0.569187 0.369687 0.0403594 +0.133355 0.723682 0.301546 0.0253954 +0.701308 0.687209 0.606802 0.0578792 +0.457599 0.23767 0.16794 0.0560995 +0.268379 0.00918446 0.992949 0.0546149 +0.245548 0.0499193 0.242448 0.030146 +0.20272 0.176961 0.181948 0.0366172 +0.884051 0.701489 0.441833 0.0782705 +0.935319 0.741255 0.755999 0.038481 +0.603378 0.974878 0.487254 0.0440824 +0.594915 0.110882 0.159449 0.0375641 +0.243172 0.802119 0.744006 0.0276907 +0.408222 0.579093 0.552614 0.0489179 +0.228555 0.354672 0.785986 0.0312958 +0.856584 0.653507 0.325309 0.0507047 +0.597334 0.206196 0.0957485 0.0771024 +0.283889 0.712962 0.213986 0.0306821 +0.898544 0.685006 0.236737 0.0522403 +0.612952 0.0954079 0.632879 0.0270156 +0.793136 0.475629 0.0530304 0.0462767 +0.571746 0.454981 0.791377 0.0445645 +0.909637 0.600943 0.19946 0.0273007 +0.230215 0.335865 0.371416 0.0384078 +0.759102 0.310971 0.324474 0.0438398 +0.390615 0.605689 0.15239 0.051238 +0.704326 0.753768 0.268878 0.0332777 +0.385959 0.403218 0.886379 0.0303753 +0.867717 0.155947 0.880571 0.0475007 +0.559515 0.282195 0.614349 0.038274 +0.282124 0.668662 0.0279222 0.0347093 +0.996496 0.074498 0.256737 0.0801265 +0.0259418 0.824566 0.787878 0.0284572 +0.19044 0.738878 0.089335 0.0775039 +0.354784 0.953639 0.162629 0.0672973 +0.339686 0.759322 0.193285 0.0417948 +0.889729 0.189187 0.472767 0.0411012 +0.202161 0.822862 0.671871 0.0578405 +0.75915 0.558727 0.679505 0.0445107 +0.385518 0.746286 0.517993 0.0557014 +0.537305 0.755397 0.810647 0.0555868 +0.773323 0.896908 0.832819 0.0527755 +0.63117 0.967736 0.939447 0.042208 +0.684104 0.283574 0.56832 0.0370857 +0.489102 0.29147 0.253914 0.0500993 +0.98902 0.686564 0.674521 0.0561929 +0.485016 0.930042 0.770817 0.0274736 +0.848196 0.112973 0.484066 0.0464272 +0.322057 0.258294 0.100725 0.0300688 +0.822778 0.957931 0.637732 0.0268777 +0.331599 0.957369 0.702546 0.0410657 +0.571942 0.639819 0.815646 0.0651733 +0.92395 0.653213 0.141702 0.0511428 +0.67738 0.504931 0.146627 0.0534621 +0.333582 0.589735 0.417686 0.0501387 +0.755897 0.269502 0.831411 0.0347693 +0.314126 0.655951 0.611588 0.0367377 +0.888558 0.573205 0.774852 0.04219 +0.917248 0.783021 0.499675 0.0178553 +0.462579 0.709217 0.126554 0.0543539 +0.249306 0.869238 0.531904 0.032742 +0.331156 0.370324 0.605939 0.0354713 +0.299021 0.497821 0.430723 0.0376934 +0.728313 0.868578 0.19144 0.0339572 +0.969073 0.516358 0.43972 0.0268611 +0.908145 0.855793 0.233966 0.03672 +0.171295 0.164906 0.492718 0.0405356 +0.835372 0.0563031 0.134669 0.03441 +0.858229 0.777248 0.874069 0.0407599 +0.454722 0.965879 0.645191 0.0296256 +0.300597 0.620075 0.933849 0.0514327 +0.406939 0.586467 0.98472 0.0319148 +0.496933 0.865768 0.749355 0.0390708 +0.565188 0.842322 0.729512 0.0357769 +0.0867276 0.379118 0.114077 0.0371997 +0.899049 0.279623 0.511086 0.0575595 +0.0360391 0.798952 0.538614 0.0296033 +0.208456 0.584214 0.130253 0.0513023 +0.0175865 0.422564 0.0729844 0.054215 +0.894704 0.38233 0.631404 0.0235709 +0.980625 0.261192 0.35711 0.0490317 +0.616941 0.263898 0.575755 0.0332937 +0.326896 0.122111 0.344022 0.0388113 +0.780194 0.954605 0.277845 0.0389321 +0.275456 0.962915 0.236888 0.026031 +0.169237 0.172421 0.910605 0.0332528 +0.451137 0.769229 0.652631 0.0311484 +0.788776 0.355068 0.778906 0.0606007 +0.051865 0.779589 0.127684 0.0306529 +0.786477 0.262534 0.54101 0.060169 +0.378441 0.382782 0.070712 0.0376256 +0.593938 0.718647 0.920339 0.028881 +0.384341 0.945156 0.254974 0.0248785 +0.758222 0.890776 0.057019 0.040147 +0.999841 0.548597 0.0212119 0.083188 +0.659783 0.841254 0.258114 0.0654834 +0.415835 0.666525 0.390407 0.0525849 +0.0264194 0.111922 0.847742 0.039674 +0.31878 0.326411 0.540293 0.0315092 +0.252682 0.296689 0.254669 0.0386987 +0.483474 0.278023 0.336693 0.0339451 +0.312515 0.180723 0.252916 0.0289338 +0.151438 0.874828 0.0897261 0.0310137 +0.152701 0.563796 0.502507 0.0390074 +0.100286 0.311386 0.879481 0.0464507 +0.271135 0.00920568 0.193371 0.0370391 +0.908098 0.95727 0.536746 0.0223119 +0.859545 0.46665 0.581479 0.0805229 +0.076527 0.231432 0.871095 0.0373791 +0.264035 0.228661 0.294603 0.040997 +0.843457 0.8361 0.544993 0.0325661 +0.412815 0.946331 0.587577 0.0279327 +0.878375 0.355889 0.481605 0.0267791 +0.946594 0.311012 0.974744 0.0326661 +0.947605 0.550931 0.640879 0.0353333 +0.343085 0.0435256 0.868761 0.0374589 +0.166311 0.460909 0.534484 0.0444463 +0.103623 0.0836423 0.225056 0.03196 +0.676459 0.664934 0.900238 0.0716129 +0.458335 0.086553 0.309138 0.0313377 +0.736818 0.379516 0.171923 0.051969 +0.872022 0.534179 0.163354 0.0452758 +0.931837 0.858315 0.608931 0.0276757 +0.798821 0.611557 0.593549 0.0272634 +0.170491 0.710158 0.86306 0.0692185 +0.899537 0.628654 0.949091 0.064025 +0.683578 0.914613 0.518551 0.0440713 +0.59966 0.782223 0.75405 0.0327934 +0.906943 0.740382 0.321144 0.0490593 +0.936971 0.316981 0.752178 0.0345918 +0.0687998 0.756149 0.0614835 0.0404359 +0.933386 0.893476 0.462098 0.040805 +0.535064 0.588606 0.177755 0.0387173 +0.286364 0.629251 0.23683 0.015917 +0.725347 0.874918 0.110905 0.0249138 +0.310144 0.0352835 0.656837 0.0233784 +0.955641 0.119577 0.419929 0.0683606 +0.177981 0.0268394 0.0494086 0.0534185 +0.310534 0.239735 0.629011 0.0463661 +0.0764343 0.718291 0.782975 0.038172 +0.281902 0.0953466 0.621981 0.046001 +0.978605 0.66228 0.297902 0.0408348 +0.919966 0.731897 0.833311 0.0408941 +0.00625005 0.480994 0.834724 0.0677248 +0.735093 0.408107 0.333932 0.0486528 +0.451139 0.486956 0.55488 0.0527498 +0.200406 0.545933 0.95564 0.0464773 +0.130971 0.886678 0.628853 0.0275564 +0.398634 0.28566 0.335307 0.0250805 +0.0398389 0.322007 0.945096 0.0335315 +0.0502095 0.579737 0.765942 0.0603907 +0.620648 0.898918 0.441397 0.0424102 +0.351003 0.498656 0.879236 0.0289499 +0.756671 0.365594 0.259963 0.0393497 +0.869145 0.883358 0.00872702 0.0474228 +0.550988 0.131449 0.568281 0.0562414 +0.00230604 0.635247 0.501495 0.0506369 +0.443483 0.390833 0.91736 0.0356127 +0.140248 0.188488 0.986744 0.0376553 +0.356848 0.705944 0.811928 0.0430153 +0.542447 0.424266 0.0847509 0.029114 +0.29865 0.170855 0.67169 0.0355307 +0.595545 0.796221 0.185268 0.0232282 +0.232712 0.626678 0.963124 0.022789 +0.539371 0.369815 0.728725 0.0341882 +0.16608 0.935203 0.117917 0.0372082 +0.288582 0.515884 0.00792882 0.0603501 +0.5961 0.757693 0.141176 0.0254382 +0.749296 0.889062 0.393224 0.0953145 +0.0123291 0.279954 0.617304 0.0358262 +0.148904 0.489338 0.838536 0.045038 +0.120644 0.0894118 0.967941 0.0642239 +0.510207 0.574344 0.527718 0.0561697 +0.613623 0.0866475 0.697253 0.0353059 +0.251014 0.533934 0.273575 0.0304216 +0.239875 0.552333 0.868744 0.0365102 +0.289184 0.197058 0.0277238 0.0707264 +0.0135747 0.270546 0.229824 0.0375919 +0.291071 0.514023 0.119834 0.0478333 +0.96844 0.485293 0.122876 0.0398037 +0.465573 0.858254 0.811117 0.0306033 +0.434766 0.0297647 0.0860781 0.0670616 +0.598234 0.490453 0.0794753 0.0329036 +0.114825 0.629261 0.864269 0.0289874 +0.388457 0.676849 0.120553 0.0267495 +0.118763 0.377762 0.977414 0.0683615 +0.252149 0.671487 0.899242 0.0242966 +0.428144 0.417859 0.356731 0.0307632 +0.250991 0.109369 0.20236 0.0417633 +0.797543 0.487194 0.375054 0.0489462 +0.00785727 0.115542 0.493049 0.0215798 +0.0115686 0.287799 0.887511 0.0391695 +0.962537 0.234295 0.134342 0.0391978 +0.57679 0.338906 0.669283 0.0425492 +0.354715 0.164709 0.941748 0.0268352 +0.7143 0.23185 0.949912 0.0661416 +0.849605 0.116473 0.952845 0.0368196 +0.482705 0.382622 0.539447 0.0286371 +0.033736 0.415446 0.641129 0.0403581 +0.923 0.632795 0.517314 0.0298854 +0.346192 0.914996 0.645426 0.0313775 +0.212817 0.9383 0.473832 0.0645895 +0.41125 0.00308085 0.236885 0.0270674 +0.515421 0.898701 0.813569 0.0336363 +0.174668 0.0934704 0.653187 0.0379566 +0.877357 0.810005 0.0359966 0.0308997 +0.583537 0.738079 0.248077 0.0631999 +0.586477 0.438278 0.0544158 0.0261596 +0.130483 0.548292 0.598109 0.060359 +0.138618 0.923867 0.843792 0.0444564 +0.124988 0.184624 0.138229 0.0494853 +0.467625 0.679468 0.782453 0.0512561 +0.152078 0.996293 0.949202 0.0358281 +0.013031 0.0342143 0.711944 0.056647 +0.416111 0.256978 0.751306 0.0254827 +0.721014 0.219189 0.620568 0.0516055 +0.517444 0.635822 0.280026 0.0315634 +0.375251 0.135829 0.126841 0.0231734 +0.308568 0.40922 0.53031 0.0525218 +0.821789 0.228777 0.868455 0.036579 +0.00694257 0.994083 0.532325 0.0496413 +0.193518 0.482122 0.249347 0.0506789 +0.332674 0.307016 0.459568 0.0526675 +0.000981402 0.0577834 0.136637 0.0412142 +0.721638 0.428485 0.564723 0.0288081 +0.486014 0.136639 0.855586 0.0372669 +0.142108 0.409874 0.723388 0.0481242 +0.732589 0.852791 0.584624 0.0233722 +0.605473 0.409348 0.967497 0.0673957 +0.979559 0.22529 0.198031 0.027339 +0.366773 0.824294 0.735596 0.0981645 +0.359326 0.510606 0.287548 0.0444792 +0.915505 0.454109 0.184911 0.0475055 +0.263309 0.451053 0.0717766 0.0340866 +0.639718 0.475227 0.817824 0.0311292 +0.954151 0.594437 0.713845 0.0498706 +0.0740118 0.479709 0.6405 0.0354853 +0.765506 0.370749 0.952906 0.0366326 +0.749728 0.821528 0.108977 0.033811 +0.28356 0.107445 0.924446 0.0661253 +0.303096 0.940528 0.455917 0.0261592 +0.977641 0.17473 0.325719 0.043001 +0.0640955 0.859261 0.637995 0.0452968 +0.135557 0.451662 0.20154 0.0303935 +0.817093 0.793859 0.151218 0.0358785 +0.591257 0.17696 0.707583 0.0277417 +0.645709 0.765292 0.451382 0.0762422 +0.0614111 0.103669 0.0733934 0.057563 +0.301086 0.358721 0.811258 0.0431728 +0.546227 0.0624158 0.415237 0.0465221 +0.826895 0.581421 0.901229 0.0349625 +0.503411 0.535501 0.651998 0.0677529 +0.125902 0.0219953 0.706209 0.0567656 +0.765977 0.515479 0.186335 0.0441982 +0.0421355 0.192512 0.265262 0.0466917 +0.273547 0.367492 0.2074 0.0421551 +0.364757 0.529338 0.834251 0.0242906 +0.284525 0.730549 0.908996 0.0437594 +0.8603 0.0286621 0.878763 0.0298667 +0.933536 0.361665 0.13182 0.0606133 +0.864348 0.150857 0.297554 0.0761568 +0.400002 0.352062 0.396502 0.0499948 +0.639976 0.899156 0.618586 0.0513284 +0.0216107 0.767948 0.816365 0.0295823 +0.745747 0.898039 0.755023 0.0297713 +0.309952 0.519742 0.842376 0.0301134 +0.976632 0.304867 0.817437 0.0248115 +0.749078 0.683385 0.786832 0.0429821 +0.916468 0.758192 0.178407 0.0430482 +0.317356 0.763227 0.974737 0.0366617 +0.648856 0.95647 0.565268 0.0274525 +0.0836226 0.678758 0.0950223 0.0452022 +0.783014 0.774839 0.480832 0.0525332 +0.871195 0.2286 0.74686 0.0313674 +0.972333 0.625624 0.222697 0.0430629 +0.396351 0.471575 0.902426 0.0287361 +0.6258 0.00391348 0.555842 0.0261327 +0.148346 0.406598 0.15298 0.0370778 +0.204421 0.0544748 0.199116 0.0297696 +0.249983 0.462982 0.768876 0.0804317 +0.238154 0.0448269 0.69253 0.0511755 +0.968369 0.795145 0.562045 0.0421101 +0.211893 0.588772 0.237785 0.0458583 +0.586302 0.928559 0.225979 0.0452946 +0.505092 0.931463 0.159407 0.0301941 +0.692916 0.936996 0.00600364 0.0536423 +0.0589582 0.716046 0.861677 0.0424784 +0.0612987 0.828561 0.0313128 0.0383678 +0.207312 0.885367 0.310449 0.0326454 +0.153396 0.105964 0.573872 0.0451062 +0.741283 0.840387 0.724117 0.0357951 +0.324877 0.684041 0.419784 0.0445911 +0.27476 0.416635 0.00534149 0.03989 +0.151219 0.269376 0.0640863 0.0661351 +0.847857 0.469953 0.7697 0.0689136 +0.510268 0.264806 0.999905 0.0225046 +0.725598 0.71226 0.516006 0.0393908 +0.987684 0.794956 0.459355 0.062381 +0.559979 0.679772 0.485753 0.0496338 +0.664613 0.842951 0.142218 0.0505254 +0.100499 0.524966 0.443032 0.0364744 +0.350953 0.368467 0.858566 0.0262517 +0.45364 0.0645374 0.239865 0.0370123 +0.0485511 0.936125 0.300061 0.0353365 +0.337742 0.0231515 0.938037 0.0349487 +0.840909 0.475627 0.452921 0.0408759 +0.526407 0.0622555 0.722434 0.0410633 +0.366983 0.401435 0.24027 0.0371918 +0.722487 0.827801 0.797527 0.0399911 +0.395336 0.953654 0.663376 0.0336743 +0.0342908 0.981297 0.366577 0.0412156 +0.985052 0.73436 0.266048 0.0382325 +0.425585 0.00590192 0.746555 0.0691061 +0.629567 0.616804 0.352128 0.103091 +0.078834 0.670421 0.64681 0.0391756 +0.464334 0.307975 0.679995 0.074574 +0.986176 0.0776001 0.940996 0.060906 +0.649439 0.411599 0.78064 0.0432054 +0.447249 0.448014 0.88513 0.0299565 +0.904255 0.966317 0.221476 0.066349 +0.190015 0.335139 0.729082 0.0401528 +0.373539 0.336143 0.515543 0.0225088 +0.437494 0.147844 0.458628 0.0270593 +0.465594 0.157736 0.0781741 0.0643634 +0.952881 0.707576 0.547516 0.0479972 +0.111766 0.720886 0.977744 0.0600722 +0.93048 0.626189 0.340078 0.0294412 +0.635956 0.0298036 0.794651 0.0632447 +0.299262 0.574092 0.324487 0.0504029 +0.880568 0.25479 0.0790395 0.0338074 +0.11961 0.81402 0.263792 0.0340143 +0.135618 0.803928 0.464253 0.0402053 +0.386374 0.0798154 0.914018 0.0349228 +0.466246 0.405301 0.658615 0.024366 +0.835743 0.571332 0.0717393 0.0307205 +0.991649 0.880697 0.523276 0.044639 +0.976781 0.197551 0.513835 0.0555144 +0.250682 0.479758 0.658656 0.0310594 +0.940707 0.0276485 0.598567 0.0429915 +0.267212 0.284087 0.182187 0.0362916 +0.682976 0.766956 0.0878854 0.0442253 +0.392205 0.686577 0.727909 0.0420919 +0.584229 0.975215 0.0172434 0.0489605 +0.939109 0.241228 0.694183 0.055504 +0.208582 0.438481 0.965725 0.0402734 +0.805122 0.191033 0.468075 0.0436557 +0.0154984 0.58286 0.851049 0.0267722 +0.696603 0.429101 0.123107 0.0282258 +0.856669 0.800589 0.37344 0.0452128 +0.569857 0.82614 0.221965 0.0296646 +0.124829 0.473713 0.681566 0.0301254 +0.0148122 0.937305 0.447386 0.0528306 +0.604187 0.533441 0.530921 0.0289388 +0.880042 0.0062386 0.0687681 0.0359922 +0.40117 0.828651 0.349636 0.0411724 +0.293261 0.452735 0.600352 0.0313456 +0.998547 0.721253 0.108041 0.0507827 +0.392793 0.753757 0.953511 0.0397787 +0.726134 0.505473 0.801638 0.0618468 +0.380704 0.108258 0.00344015 0.0590851 +0.448954 0.927581 0.828124 0.0402803 +0.662271 0.121398 0.121721 0.032505 +0.784095 0.14922 0.391235 0.044929 +0.389297 0.284004 0.574062 0.0574368 +0.867518 0.97166 0.584294 0.0418326 +0.497479 0.0400965 0.363723 0.0277973 +0.371028 0.915878 0.455969 0.0431503 +0.211006 0.984067 0.879521 0.0562441 +0.570928 0.0708978 0.273395 0.0851397 +0.18681 0.389689 0.209676 0.0334898 +0.305225 0.604874 0.55304 0.0412881 +0.818266 0.453808 0.264575 0.0683132 +0.751814 0.579974 0.87378 0.0449915 +0.653218 0.35616 0.661213 0.0295761 +0.845382 0.0664555 0.622021 0.0625688 +0.555387 0.230456 0.519778 0.0506727 +0.17221 0.0679479 0.488745 0.0500029 +0.760739 0.538814 0.299487 0.0401036 +0.255051 0.662142 0.28998 0.0539914 +0.034653 0.841985 0.137435 0.0348042 +0.670713 0.511742 0.481939 0.0288189 +0.567923 0.895131 0.776312 0.0268042 +0.800579 0.186578 0.698107 0.0641802 +0.567662 0.0217846 0.591851 0.043904 +0.385192 0.241816 0.652955 0.0320645 +0.568372 0.712072 0.158986 0.0308401 +0.535068 0.297867 0.97276 0.0269417 +0.558947 0.633041 0.587896 0.0410025 +0.213499 0.201264 0.114192 0.0346295 +0.111418 0.506561 0.507827 0.0317634 +0.551 0.127406 0.669013 0.0445715 +0.0744426 0.786094 0.45218 0.0246502 +0.96074 0.808602 0.818023 0.0436351 +0.47775 0.47478 0.431632 0.0739256 +0.214553 0.33828 0.147481 0.0468613 +0.587652 0.442105 0.686194 0.0625881 +0.154884 0.647331 0.284012 0.0474396 +0.999442 0.947992 0.75609 0.041168 +0.205006 0.403058 0.0241933 0.0281731 +0.921534 0.520137 0.809626 0.028763 +0.77182 0.555093 0.957734 0.0448276 +0.108629 0.448488 0.261172 0.0346842 +0.291529 0.930636 0.382526 0.0487929 +0.274083 0.635301 0.819799 0.0657126 +0.0779328 0.274927 0.606454 0.0308584 +0.911015 0.0349674 0.841256 0.0328555 +0.430092 0.458727 0.968957 0.0373765 +0.824794 0.491249 0.679633 0.0264672 +0.34343 0.419856 0.329033 0.0535487 +0.151281 0.952009 0.641326 0.0419865 +0.751363 0.0344032 0.14959 0.0523993 +0.576696 0.307792 0.187256 0.061177 +0.618775 0.883849 0.361262 0.0391506 +0.698298 0.360607 0.769547 0.0282821 +0.0806696 0.783201 0.772865 0.0276567 +0.807101 0.645511 0.390605 0.0297998 +0.352365 0.203387 0.68773 0.0292839 +0.793673 0.142216 0.840006 0.0380357 +0.876588 0.565727 0.39755 0.0647267 +0.712125 0.841764 0.5086 0.0348021 +0.889768 0.487325 0.0733822 0.0531649 +0.55101 0.487564 0.561216 0.045406 +0.250751 0.267961 0.106675 0.0421345 +0.641358 0.753756 0.949757 0.035968 +0.411034 0.266291 0.273339 0.034197 +0.369732 0.504878 0.380561 0.0492892 +0.614222 0.666951 0.548958 0.0346371 +0.474508 0.274928 0.807459 0.0574968 +0.780645 0.410693 0.409692 0.0367141 +0.162644 0.491272 0.0576226 0.075236 +0.762229 0.568688 0.221017 0.0162363 +0.688626 0.33076 0.986333 0.0412615 +0.957282 0.567235 0.823825 0.0320461 +0.215031 0.0149646 0.58225 0.0651206 +0.756064 0.750699 0.592844 0.0233468 +0.854734 0.319731 0.0895236 0.0368656 +0.030286 0.687078 0.0328483 0.036585 +0.508622 0.933687 0.0746556 0.0546599 +0.227022 0.430378 0.451181 0.0630584 +0.889939 0.304206 0.932544 0.0365573 +0.996073 0.222829 0.0342757 0.0336669 +0.636417 0.519264 0.662153 0.0310773 +0.0920303 0.722615 0.208693 0.0762189 +0.891592 0.569244 0.859746 0.0428503 +0.834025 0.231262 0.408396 0.0336173 +0.285339 0.158489 0.588531 0.0255368 +0.879403 0.217234 0.817834 0.0409782 +0.197831 0.558775 0.317646 0.0406011 +0.710889 0.971403 0.903483 0.0453249 +0.00244868 0.797205 0.656727 0.0420907 +0.742346 0.622211 0.046492 0.0702879 +0.336876 0.459237 0.076248 0.0358286 +0.678083 0.812578 0.739198 0.0348807 +0.502568 0.709604 0.204356 0.0331242 +0.455794 0.589949 0.466838 0.0269611 +0.536974 0.509241 0.344717 0.0367502 +0.983691 0.918068 0.86768 0.0754331 +0.835419 0.403923 0.484338 0.0376011 +0.102984 0.592024 0.0827222 0.0445142 +0.772921 0.432949 0.863463 0.0443188 +0.431513 0.527305 0.760956 0.0472571 +0.269559 0.394584 0.359787 0.0303637 +0.602987 0.601179 0.509488 0.0421183 +0.547028 0.386177 0.229397 0.0326335 +0.604434 0.0637653 0.969294 0.053746 +0.0362922 0.493514 0.441899 0.0350311 +0.611215 0.290741 0.952897 0.0520755 +0.873398 0.692403 0.552281 0.0330611 +0.0955897 0.811148 0.96613 0.0320409 +0.738778 0.184456 0.802299 0.0408286 +0.0794721 0.480103 0.358756 0.0425724 +0.415433 0.00756914 0.542266 0.048291 +0.0843393 0.558346 0.209766 0.0882338 +0.437067 0.894573 0.632725 0.0448841 +0.942038 0.0110999 0.106645 0.0368213 +0.339176 0.66249 0.762079 0.0254347 +0.0237599 0.166046 0.160141 0.055723 +0.272896 0.794781 0.202206 0.0343483 +0.067522 0.918224 0.0846135 0.063597 +0.986633 0.620803 0.814805 0.0296985 +0.961634 0.578027 0.450432 0.0361721 +0.0854466 0.32891 0.253634 0.035406 +0.820493 0.905422 0.58555 0.0394094 +0.717882 0.207086 0.0615015 0.048219 +0.608647 0.884135 0.00680622 0.0458381 +0.923982 0.651092 0.787454 0.0373728 +0.298369 0.753886 0.834193 0.035813 +0.424418 0.957239 0.911392 0.0459779 +0.369889 0.500442 0.711803 0.0360208 +0.0161444 0.608757 0.681981 0.0202771 +0.724098 0.706142 0.309687 0.0324834 +0.450584 0.650065 0.0615262 0.0296411 +0.380815 0.616605 0.0775649 0.0250112 +0.714733 0.723489 0.374471 0.0352342 +0.163826 0.0296769 0.801277 0.0458751 +0.285913 0.934078 0.556121 0.0455574 +0.0501681 0.663954 0.731698 0.0305223 +0.0654177 0.891432 0.964113 0.0537495 +0.342447 0.547093 0.771546 0.0445932 +0.248892 0.100704 0.447258 0.0431325 +0.843833 0.754907 0.802844 0.0352627 +0.793623 0.826593 0.605191 0.0461586 +0.614333 0.462229 0.494526 0.0463799 +0.844938 0.417427 0.682574 0.032863 +0.792555 0.10373 0.135354 0.0294889 +0.871637 0.858459 0.592829 0.0272858 +0.37826 0.207292 0.231862 0.0450207 +0.907726 0.0232485 0.674226 0.0290572 +0.446263 0.075163 0.864548 0.0364873 +0.617691 0.718318 0.806913 0.0246094 +0.362694 0.0646718 0.621414 0.0404203 +0.118275 0.319686 0.572569 0.0382717 +0.954371 0.104646 0.856986 0.0329059 +0.646031 0.0252128 0.340101 0.0252113 +0.734907 0.793802 0.556628 0.0367947 +0.970231 0.235422 0.435392 0.0318371 +0.953019 0.0995304 0.590692 0.0296419 +0.388295 0.605101 0.91917 0.0387371 +0.636332 0.976891 0.393936 0.0477772 +0.72511 0.202648 0.242359 0.0194786 +0.737841 0.775151 0.340978 0.0305294 +0.039406 0.289406 0.529162 0.056709 +0.0734211 0.397391 0.200481 0.0521129 +0.77524 0.983995 0.581261 0.0513153 +0.922219 0.395047 0.0164646 0.0600072 +0.184476 0.103312 0.137603 0.0512658 +0.667228 0.149252 0.940031 0.0169532 +0.324764 0.162308 0.852089 0.0335907 +0.881473 0.851158 0.949434 0.0211667 +0.903206 0.853736 0.657081 0.028499 +0.71157 0.410418 0.994149 0.0420032 +0.884663 0.907765 0.926181 0.0400439 +0.0315263 0.719653 0.51169 0.0392634 +0.0114382 0.281378 0.756491 0.0480615 +0.34151 0.690674 0.986838 0.0407582 +0.582298 0.210682 0.597341 0.0337734 +0.176485 0.901389 0.172496 0.027835 +0.951502 0.934567 0.590165 0.0501649 +0.920278 0.754928 0.679292 0.0408734 +0.365294 0.459056 0.0121759 0.0342632 +0.0268705 0.440835 0.963344 0.0417804 +0.7885 0.645222 0.939298 0.0486673 +0.652416 0.48034 0.746306 0.0336915 +0.366542 0.655944 0.553419 0.0385113 +0.536247 0.964461 0.257982 0.0241221 +0.0259353 0.530341 0.914529 0.0281468 +0.179045 0.0987823 0.268531 0.056402 +0.923414 0.919355 0.76775 0.0408986 +0.538275 0.505041 0.741603 0.0331047 +0.382159 0.676024 0.0558488 0.0382659 +0.441976 0.455106 0.675699 0.0336116 +0.978377 0.703441 0.194623 0.0398837 +0.831251 0.165203 0.780096 0.0259307 +0.00364 0.92026 0.233324 0.0438282 +0.467971 0.315008 0.387975 0.0278863 +0.685951 0.49744 0.328845 0.0307398 +0.0158312 0.173466 0.905644 0.0454873 +0.494032 0.752951 0.50673 0.0375593 +0.263785 0.498234 0.864375 0.0222977 +0.239463 0.124497 0.537115 0.0502986 +0.380107 0.838709 0.162076 0.0503912 +0.82886 0.315167 0.656725 0.0738324 +0.0675429 0.201091 0.050327 0.0427404 +0.986868 0.483968 0.605124 0.0501303 +0.299252 0.691079 0.697256 0.0558879 +0.229132 0.355603 0.965813 0.0448059 +0.796856 0.108726 0.768505 0.040846 +0.896322 0.738621 0.083714 0.0557231 +0.0671488 0.539157 0.102483 0.0220993 +0.909908 0.0463438 0.977915 0.0289417 +0.173895 0.869224 0.239998 0.0469829 +0.0787844 0.205243 0.422593 0.081558 +0.293698 0.807183 0.121087 0.0503083 +0.783852 0.856631 0.516396 0.0366591 +0.492898 0.996042 0.992896 0.0478275 +0.440323 0.608849 0.304519 0.0537302 +0.811262 0.59727 0.987329 0.0177226 +0.100607 0.194481 0.667766 0.0449606 +0.521655 0.840888 0.022524 0.0525739 +0.918465 0.437695 0.83135 0.0302162 +0.133958 0.853546 0.366657 0.0222612 +0.549136 0.586441 0.921747 0.0490256 +0.0832033 0.510056 0.703921 0.0218471 +0.164697 0.23677 0.705296 0.0405048 +0.9332 0.289731 0.606825 0.0445898 +0.46791 0.751609 0.349668 0.0508597 +0.744209 0.523271 0.0193919 0.0298499 +0.499672 0.646739 0.350333 0.0302209 +0.40252 0.544405 0.878276 0.0356364 +0.394489 0.294922 0.907759 0.0723599 +0.710389 0.0661119 0.389871 0.0658473 +0.241389 0.732512 0.772107 0.0473955 +0.940823 0.425529 0.656801 0.0376741 +0.377912 0.538888 0.0735657 0.0528627 +0.504579 0.0484129 0.514794 0.052888 +0.21807 0.828948 0.395531 0.0700083 +0.46496 0.832098 0.120739 0.0442256 +0.0441994 0.036832 0.00862465 0.0370856 +0.103997 0.820436 0.0985555 0.0416986 +0.472064 0.19164 0.264351 0.0517102 +0.204786 0.0495086 0.940763 0.0327647 +0.103964 0.0504841 0.302309 0.0390487 +0.28533 0.88497 0.145955 0.0317849 +0.309861 0.478293 0.674882 0.0303205 +0.363496 0.494526 0.144771 0.0312066 +0.940978 0.854591 0.55939 0.0228392 +0.421612 0.381454 0.488424 0.0489023 +0.944424 0.531099 0.275253 0.0686327 +0.484188 0.564546 0.847219 0.0522262 +0.487402 0.984988 0.387992 0.0329924 +0.780091 0.607708 0.752385 0.0457618 +0.22637 0.817536 0.819342 0.0510213 +0.351171 0.224305 0.437923 0.0348068 +0.606494 0.814031 0.0685926 0.0476324 +0.0735924 0.386593 0.528689 0.046317 +0.492501 0.441407 0.933377 0.0365202 +0.275267 0.635579 0.202237 0.0209592 +0.238954 0.225223 0.41306 0.0801348 +0.537638 0.389032 0.143916 0.0388423 +0.733012 0.519545 0.0815342 0.0334029 +0.0537787 0.887019 0.781027 0.0402609 +0.749137 0.212221 0.154676 0.0497722 +0.822029 0.988922 0.744158 0.0839717 +0.340174 0.21921 0.302528 0.0361348 +0.246422 0.274448 0.692604 0.0503782 +0.020658 0.156694 0.746781 0.0709193 +0.124936 0.34373 0.791409 0.0453778 +0.539962 0.656581 0.223979 0.0323057 +0.297447 0.531048 0.382593 0.0208131 +0.488031 0.377957 0.791846 0.0475822 +0.172883 0.0140532 0.423063 0.0349631 +0.502975 0.943941 0.56179 0.0616785 +0.476361 0.36677 0.428283 0.0337416 +0.476334 0.829616 0.952721 0.0314109 +0.474474 0.993812 0.196474 0.0485386 +0.698998 0.295529 0.403672 0.0567752 +0.529496 0.659155 0.157709 0.0348353 +0.504104 0.396547 0.486109 0.0249825 +0.579664 0.675727 0.0373729 0.096589 +0.37825 0.453693 0.812835 0.0554769 +0.378158 0.709608 0.163111 0.0279343 +0.151949 0.188553 0.845394 0.0361128 +0.908731 0.217985 0.395729 0.0433101 +0.891709 0.518298 0.492238 0.0274859 +0.111246 0.974331 0.213411 0.0782339 +0.152132 0.498305 0.39339 0.0399519 +0.575462 0.350763 0.444815 0.0432159 +0.000202064 0.86589 0.037123 0.0334521 +0.639929 0.742277 0.634838 0.0292177 +0.726025 0.930271 0.224489 0.0360671 +0.79744 0.302492 0.243373 0.0375866 +0.240915 0.179497 0.839896 0.0528664 +0.586052 0.229067 0.319094 0.0801638 +0.160903 0.135881 0.75258 0.0709799 +0.28669 0.76761 0.61722 0.0555592 +0.386181 0.714765 0.626709 0.0574945 +0.973931 0.309386 0.282966 0.0392466 +0.704065 0.470432 0.404514 0.0504873 +0.36903 0.189644 0.370309 0.0432448 +0.309127 0.872658 0.272614 0.0767134 +0.159053 0.411207 0.814151 0.0335804 +0.586256 0.215018 0.205871 0.0339272 +0.181608 0.301697 0.974377 0.0275538 +0.199739 0.236496 0.976182 0.0395169 +0.121279 0.0256993 0.880398 0.0426744 +0.0988272 0.765001 0.671472 0.0605903 +0.73433 0.534599 0.365045 0.0306988 +0.835826 0.0772126 0.542087 0.0186496 +0.236253 0.196815 0.719136 0.0315344 +0.290045 0.420495 0.901848 0.0647978 +0.596852 0.358384 0.0781973 0.0547767 +0.858538 0.565518 0.68123 0.0551239 +0.623878 0.269676 0.454435 0.0375028 +0.872172 0.862595 0.840186 0.0521191 +0.283073 0.591122 0.0759607 0.0412346 +0.752534 0.175733 0.329665 0.0291655 +0.375085 0.191131 0.890031 0.0347068 +0.837384 0.36181 0.189891 0.0517135 +0.172474 0.159115 0.0584824 0.0462951 +0.850605 0.468533 0.975531 0.0504654 +0.828825 0.944744 0.510441 0.0457791 +0.455699 0.701426 0.271601 0.0404448 +0.73065 0.197981 0.448528 0.0336518 +0.106134 0.49854 0.93452 0.0604127 +0.903478 0.577989 0.093234 0.0406544 +0.261668 0.764575 0.70569 0.024135 +0.630365 0.162602 0.45479 0.0695332 +0.476689 0.623172 0.689707 0.0313546 +0.971763 0.380131 0.565324 0.0620944 +0.223262 0.564793 0.428218 0.0634555 +0.346012 0.0120648 0.433007 0.0588545 +0.760486 0.757378 0.857947 0.0602771 +0.404162 0.0443883 0.18623 0.0386778 +0.176089 0.2502 0.894694 0.0464324 +0.312347 0.753403 0.284125 0.0431394 +0.250034 0.128845 0.697857 0.0338459 +0.565364 0.995818 0.397279 0.0203107 +0.512387 0.781276 0.175999 0.0445767 +0.173344 0.703212 0.325474 0.0244532 +0.507208 0.443278 0.743266 0.0360515 +0.685802 0.855526 0.0545575 0.0393374 +0.306736 0.212904 0.917425 0.0420787 +0.158305 0.526474 0.772155 0.0311341 +0.302752 0.0755398 0.385339 0.0274528 +0.603891 0.559566 0.112962 0.0441026 +0.692219 0.451049 0.244605 0.0593354 +0.578164 0.125409 0.766836 0.0518814 +0.554182 0.948538 0.33933 0.0548596 +0.0105146 0.550281 0.392836 0.0443045 +0.980228 0.175531 0.230054 0.0256296 +0.681364 0.927341 0.102508 0.0440299 +0.935634 0.401624 0.277461 0.0607776 +0.245794 0.812656 0.0583439 0.0197981 +0.462218 0.344889 0.854079 0.0274676 +0.395484 0.749755 0.283934 0.0336196 +0.385325 0.743079 0.0269592 0.0348164 +0.998422 0.970225 0.155892 0.0484727 +0.721504 0.0981203 0.0955572 0.0363057 +0.987635 0.599834 0.337308 0.0335558 +0.777571 0.871207 0.972464 0.0466652 +0.406147 0.0257129 0.884308 0.0298903 +0.455079 0.00211389 0.305063 0.0532624 +0.742284 0.500721 0.901114 0.0390439 +0.785042 0.70373 0.352834 0.0409367 +0.688548 0.898966 0.799593 0.0383012 +0.49293 0.763714 0.274028 0.0297259 +0.384052 0.868326 0.41043 0.0239661 +0.861524 0.784836 0.510298 0.0319179 +0.420515 0.857562 0.878807 0.050715 +0.0734846 0.0688039 0.392815 0.0581932 +0.715446 0.0777535 0.936592 0.0524876 +0.287142 0.0166761 0.89965 0.0280373 +0.248549 0.0234936 0.397167 0.0455672 +0.91489 0.518798 0.938188 0.0392772 +0.283796 0.870725 0.624756 0.0478714 +0.42937 0.230281 0.333507 0.0382816 +0.393093 0.615601 0.823048 0.0549605 +0.959248 0.563476 0.159068 0.0468383 +0.666744 0.136544 0.0128609 0.0515035 +0.222996 0.951617 0.387787 0.023073 +0.122743 0.371178 0.324218 0.0313988 +0.54435 0.218113 0.667561 0.0463903 +0.364778 0.21087 0.737976 0.0230105 +0.912934 0.131675 0.0659722 0.0937147 +0.14477 0.646205 0.141649 0.0383011 +0.967568 0.409403 0.721571 0.0247412 +0.0927636 0.406793 0.865344 0.0502917 +0.392806 0.927691 0.358628 0.0553065 +0.42124 0.483697 0.0303886 0.0295242 +0.527993 0.985592 0.815854 0.0541892 +0.859749 0.645277 0.738941 0.0433317 +0.570059 0.818138 0.326468 0.0496555 +0.71696 0.33224 0.511256 0.0426746 +0.740975 0.0664929 0.833154 0.0544739 +0.604503 0.536806 0.602952 0.0384279 +0.321328 0.157348 0.788415 0.030134 +0.875825 0.955753 0.426259 0.051261 +0.264806 0.144109 0.761943 0.0336686 +0.980924 0.343436 0.677114 0.0559204 +0.425978 0.366296 0.127755 0.0239908 +0.285203 0.779727 0.0373906 0.0356677 +0.7236 0.192228 0.539181 0.0341698 +0.223432 0.380121 0.289551 0.0549003 +0.122758 0.872099 0.153719 0.0361732 +0.43861 0.918358 0.763708 0.0209062 +0.817439 0.564494 0.818925 0.04193 +0.210961 0.184482 0.628897 0.0629902 +0.0595963 0.999181 0.303805 0.0287881 +0.604776 0.758543 0.871347 0.035224 +0.833141 0.0366742 0.270819 0.0451952 +0.741396 0.424715 0.927634 0.0276503 +0.398515 0.29509 0.0932397 0.0551126 +0.877877 0.4771 0.347628 0.0365381 +0.367608 0.485208 0.617286 0.0515327 +0.196724 0.348709 0.868687 0.05752 +0.226177 0.847299 0.170671 0.0425731 +0.187714 0.254087 0.166325 0.0434933 +0.677171 0.370944 0.0469126 0.0323308 +0.188026 0.618244 0.0291106 0.0573495 +0.752067 0.943366 0.507298 0.0310557 +0.573572 0.662508 0.909862 0.0317499 +0.0940678 0.619951 0.528223 0.0461545 +0.289861 0.587052 0.48744 0.0284042 +0.227273 0.102636 0.0172705 0.042538 +0.600155 0.468923 0.395562 0.0538181 +0.376566 0.395933 0.965208 0.0452198 +0.557942 0.398274 0.535156 0.047868 +0.200184 0.64302 0.512609 0.0539066 +0.705183 0.94124 0.583375 0.0281381 +0.095957 0.408135 0.367883 0.0317681 +0.236514 0.743982 0.186436 0.030095 +0.15277 0.852787 0.786637 0.0327913 +0.689575 0.687679 0.12058 0.0417824 +0.971536 0.375338 0.909365 0.0595384 +0.925398 0.151795 0.939434 0.0350176 +0.206241 0.531523 0.657214 0.0371806 +0.374368 0.832453 0.453307 0.0327712 +0.833414 0.700289 0.158749 0.0494292 +0.961247 0.299081 0.553105 0.0167281 +0.795194 0.973498 0.863279 0.0325018 +0.735551 0.620712 0.525461 0.052623 +0.85323 0.826336 0.650723 0.0288492 +0.423854 0.0057172 0.38513 0.0327531 +0.7478 0.624219 0.273596 0.03904 +0.181459 0.80043 0.571574 0.0469985 +0.0873135 0.554768 0.855106 0.0385121 +0.214339 0.614854 0.904406 0.0398616 +0.0140433 0.627803 0.120722 0.0447875 +0.672093 0.980804 0.182765 0.0488043 +0.659552 0.655571 0.486266 0.0397199 +0.665928 0.127579 0.573088 0.0588638 +0.382275 0.453866 0.509441 0.0361432 +0.760403 0.472483 0.972542 0.039872 +0.0841486 0.655736 0.817299 0.0330461 +0.137313 0.943969 0.355065 0.0689639 +0.0700236 0.992467 0.804084 0.0550774 +0.441764 0.235415 0.0949674 0.0186061 +0.954459 0.46113 0.958578 0.0335728 +0.298355 0.192709 0.161506 0.0624382 +0.450279 0.333931 0.54145 0.0278569 +0.888901 0.356806 0.589998 0.0246846 +0.503525 0.326041 0.490825 0.0456832 +0.464361 0.760942 0.0366231 0.0467878 +0.608384 0.526171 0.225891 0.0537494 +0.924893 0.864681 0.120223 0.0785918 +0.949891 0.489475 0.378431 0.0427591 +0.986838 0.490436 0.191997 0.0319083 +0.731357 0.135378 0.875037 0.0244914 +0.700099 0.683529 0.243388 0.0415627 +0.0958441 0.31352 0.690806 0.0636166 +0.655711 0.197956 0.778849 0.0359196 +0.892777 0.790912 0.814808 0.0266654 +0.604606 0.264847 0.856282 0.0452138 +0.457954 0.509764 0.917222 0.0404542 +0.431362 0.0768921 0.457814 0.0441614 +0.731027 0.763662 0.179756 0.0595046 +0.16653 0.0520236 0.35531 0.0429638 +0.695587 0.803421 0.648167 0.0545328 +0.0910972 0.130693 0.313326 0.0429291 +0.153617 0.442227 0.328054 0.0461633 +0.0603652 0.589815 0.921054 0.0408831 +0.65692 0.270291 0.0248016 0.0355456 +0.976278 0.888327 0.682703 0.0562087 +0.608229 0.0293959 0.711895 0.024034 +0.85167 0.356218 0.540545 0.0371998 +0.455245 0.406259 0.719903 0.0318922 +0.512712 0.78009 0.0999011 0.0288745 +0.348935 0.980761 0.604062 0.0463632 +0.275523 0.34978 0.732798 0.0398295 +0.605315 0.544587 0.764405 0.0480013 +0.243777 0.176684 0.241702 0.035883 +0.452277 0.164285 0.72871 0.0765489 +0.262937 0.585354 0.708821 0.0564968 +0.156196 0.775292 0.763336 0.0482034 +0.698145 0.0181687 0.50684 0.0611572 +0.254493 0.0309336 0.488312 0.0380138 +0.312814 0.210201 0.528882 0.0580524 +0.827424 0.957453 0.126521 0.0561172 +0.0936505 0.804723 0.578459 0.0406821 +0.546432 0.0585026 0.0557878 0.0502925 +0.0783327 0.0834858 0.88331 0.0293821 +0.887034 0.824603 0.45245 0.0427718 +0.83297 0.729315 0.942486 0.0465134 +0.931037 0.157116 0.19402 0.0380675 +0.180634 0.292309 0.62344 0.0467342 +0.775345 0.220963 0.236226 0.0343412 +0.994176 0.857331 0.599095 0.0347386 +0.113761 0.785112 0.344936 0.0523236 +0.0959331 0.481298 0.764348 0.0462747 +0.961413 0.962529 0.00876053 0.0741558 +0.69515 0.283219 0.230503 0.0672964 +0.393584 0.130184 0.285606 0.0502106 +0.876197 0.709806 0.769023 0.02974 +0.337299 0.655031 0.341544 0.0397736 +0.469473 0.700532 0.68713 0.046384 +0.510645 0.191802 0.818199 0.0337787 +0.839814 0.119922 0.553876 0.0242264 +0.574432 0.535962 0.461556 0.0437074 +0.36595 0.688091 0.901127 0.0484067 +0.480055 0.469133 0.818072 0.0476252 +0.0924306 0.198161 0.810536 0.033507 +0.626908 0.323577 0.610276 0.036367 +0.948917 0.22664 0.783408 0.0348852 +0.179892 0.0990967 0.868402 0.0520175 +0.0190908 0.0629969 0.596628 0.043016 +0.773655 0.373689 0.566869 0.0467767 +0.787497 0.770967 0.375507 0.0300634 +0.757074 0.213641 0.864207 0.0300178 +0.583052 0.343736 0.800494 0.0537833 +0.105031 0.633126 0.961157 0.0294953 +0.942486 0.0106145 0.363133 0.0552184 +0.534369 0.374851 0.336133 0.0754454 +0.546933 0.842436 0.796025 0.033196 +0.380806 0.594637 0.666373 0.0691258 +0.36186 0.74812 0.361994 0.0492894 +0.269021 0.29271 0.774568 0.0311913 +0.152532 0.154479 0.349567 0.03226 +0.253005 0.937719 0.0850395 0.0550371 +0.988915 0.036028 0.843536 0.0450853 +0.382145 0.225693 0.484836 0.0204174 +0.447164 0.430942 0.296252 0.0339721 +0.42257 0.138007 0.893448 0.0366282 +0.160252 0.374921 0.389111 0.0436487 +0.0482434 0.845957 0.715012 0.0212059 +0.740615 0.122605 0.489806 0.0528631 +0.337349 0.0990124 0.212631 0.0458182 +0.693194 0.249709 0.494231 0.0448816 +0.121411 0.347576 0.218927 0.0179461 +0.418447 0.188251 0.954064 0.0422126 +0.744819 0.906502 0.655015 0.0599063 +0.100297 0.253484 0.209997 0.0529896 +0.14019 0.930662 0.955294 0.0311485 +0.0578602 0.990552 0.936978 0.0492956 +0.995064 0.795563 0.196148 0.0457698 +0.809661 0.687451 0.00497543 0.0322316 +0.754905 0.497224 0.496312 0.0578159 +0.480689 0.587605 0.104943 0.052163 +0.967232 0.772533 0.955006 0.0950878 +0.640378 0.0505876 0.13364 0.0422382 +0.951064 0.551735 0.534833 0.0528596 +0.352244 0.959156 0.518134 0.0348899 +0.0970393 0.143083 0.202474 0.0319655 +0.0692308 0.0829674 0.165311 0.0369804 +0.266799 0.347426 0.492224 0.0312257 +0.549002 0.758664 0.385457 0.0342046 +0.140027 0.589776 0.913706 0.0391168 +0.733872 0.113436 0.648219 0.0247142 +0.418142 0.391261 0.592027 0.0550746 +0.302429 0.672036 0.13459 0.0605449 +0.794182 0.704974 0.256174 0.0557021 +0.60087 0.800067 0.656535 0.0187701 +0.953321 0.282556 0.0638799 0.0455256 +0.782149 0.742544 0.0723926 0.0590767 +0.641387 0.82493 0.348359 0.0252644 +0.339315 0.412177 0.153466 0.0539242 +0.0477311 0.0717216 0.494462 0.0376835 +0.375699 0.875683 0.0189331 0.0975151 +0.566335 0.989639 0.149102 0.054902 +0.262667 0.306781 0.587847 0.0436802 +0.551414 0.861478 0.609767 0.0453185 +0.984634 0.127304 0.550614 0.0213456 +0.380701 0.171767 0.472808 0.0348289 +0.765897 0.230382 0.384757 0.0385001 +0.415717 0.350024 0.295256 0.0524831 +0.217056 0.951651 0.265151 0.0398193 +0.000968982 0.430829 0.163633 0.0355991 +0.291967 0.660526 0.494621 0.0404777 +0.935084 0.23286 0.89509 0.0553053 +0.399165 0.0817017 0.371228 0.0483425 +0.70861 0.232578 0.327233 0.0427139 +0.189529 0.263609 0.794312 0.0557292 +0.0460078 0.544505 0.517246 0.0439701 +0.851237 0.543194 0.947014 0.0294591 +0.64212 0.201285 0.615519 0.0294518 +0.698296 0.952997 0.736842 0.0450782 +0.668195 0.793406 0.00872707 0.039842 +0.936557 0.240319 0.990918 0.04043 +0.0146626 0.739855 0.356655 0.0572485 +0.549926 0.322545 0.549712 0.0293694 +0.972175 0.0718838 0.778897 0.0303641 +0.403315 0.197753 0.545267 0.0345679 +0.323732 0.721837 0.0546838 0.03599 +0.805713 0.307631 0.90544 0.0519881 +0.664033 0.767965 0.819431 0.0441201 +0.843687 0.590011 0.515612 0.0602173 +0.158602 0.376182 0.0805141 0.0421789 +0.675841 0.384516 0.104361 0.0244079 +0.310049 0.562578 0.213382 0.0586225 +0.886731 0.077162 0.193628 0.046516 +0.223704 0.520118 0.580622 0.0194042 +0.824066 0.882588 0.0840988 0.0299975 +0.0820563 0.568084 0.681293 0.0223544 +0.561803 0.790122 0.940378 0.0520066 +0.306055 0.0377093 0.581704 0.0283472 +0.505633 0.431316 0.611258 0.042498 +0.548701 0.951239 0.691724 0.0762603 +0.258175 0.924392 0.000108292 0.0310885 +0.361309 0.45762 0.938736 0.0236196 +0.429362 0.718176 0.4516 0.0286272 +0.501524 0.634037 0.425925 0.0456977 +0.887232 0.703599 0.642678 0.0303125 +0.82491 0.0514652 0.367683 0.051718 +0.376757 0.758952 0.106585 0.0468273 +0.470812 0.635571 0.602808 0.0484211 +0.259759 0.792127 0.522843 0.0456003 +0.965868 0.168922 0.839911 0.0345864 +0.0983549 0.179746 0.923045 0.0390845 +0.281818 0.540212 0.613739 0.0504371 +0.892463 0.0683327 0.919766 0.0356274 +0.137587 0.380483 0.258859 0.0362674 +0.00468448 0.0284694 0.448191 0.0388986 +0.666498 0.464484 0.889145 0.0458084 +0.270945 0.604011 0.0074491 0.0295261 +0.824315 0.503138 0.892215 0.0435041 +0.97251 0.347693 0.351578 0.0380249 +0.918506 0.959685 0.679761 0.035651 +0.30356 0.202085 0.739011 0.0388433 +0.458408 0.403241 0.204633 0.0609506 +0.0245266 0.601557 0.604077 0.0583827 +0.9385 0.79256 0.256389 0.0369185 +0.539306 0.775523 0.720941 0.0363714 +0.0282537 0.738815 0.595255 0.0465328 +0.282383 0.933456 0.92909 0.0443205 +0.70318 0.555804 0.237414 0.0462375 +0.196786 0.893218 0.589367 0.0487157 +0.477582 0.754463 0.745022 0.0331508 +0.0472163 0.307289 0.316726 0.0414682 +0.848475 0.760482 0.0127684 0.030632 +0.855566 0.963201 0.327218 0.0501047 +0.196001 0.526833 0.542389 0.0282854 +0.514176 0.583022 0.30902 0.0246378 +0.262275 0.610803 0.170789 0.0211316 +0.6988 0.119396 0.732603 0.062557 +0.9775 0.0755194 0.523888 0.0373642 +0.0323348 0.675025 0.560198 0.02637 +0.518427 0.511741 0.0130218 0.0728522 +0.376741 0.645201 0.475462 0.0408404 +0.956417 0.874734 0.341891 0.0830158 +0.738857 0.748209 0.968854 0.0530326 +0.840014 0.364516 0.968792 0.0398048 +0.654462 0.537034 0.978341 0.0697949 +0.460855 0.6692 0.50814 0.0525348 +0.402601 0.773557 0.432283 0.0358422 +0.23406 0.710889 0.412779 0.0503704 +0.309571 0.319211 0.651266 0.0361729 +0.493725 0.258045 0.081677 0.039607 +0.284051 0.121809 0.276956 0.0407728 +0.22809 0.697242 0.972726 0.0476506 +0.843932 0.22086 0.962152 0.0584928 +0.377826 0.523002 0.952143 0.0451356 +0.867387 0.060789 0.434479 0.0279874 +0.207324 0.980352 0.665394 0.0252695 +0.380456 0.529325 0.476387 0.046258 +0.7503 0.903047 0.560004 0.0353261 +0.68516 0.6115 0.783432 0.0532702 +0.918005 0.0155506 0.520804 0.0389165 +0.664334 0.713144 0.72301 0.0667965 +0.235086 0.291119 0.0122034 0.0348512 +0.481982 0.328624 0.13447 0.0438376 +0.92759 0.673888 0.872296 0.0294127 +0.605795 0.198742 0.976194 0.0429829 +0.900536 0.380909 0.723756 0.0442887 +0.494755 0.0211987 0.902942 0.0455957 +0.00321646 0.49773 0.536065 0.0221596 +0.0615405 0.528514 0.669639 0.022709 +0.738223 0.117822 0.163091 0.0331204 +0.572239 0.177762 0.830199 0.0305247 +0.556812 0.487955 0.885437 0.0562204 +0.211255 0.827981 0.954537 0.0854165 +0.101848 0.0266498 0.550516 0.052328 +0.880147 0.772197 0.593574 0.0543461 +0.0412955 0.293373 0.121345 0.0601096 +0.738643 0.810195 0.0280769 0.0326969 +0.668397 0.14654 0.817262 0.0295026 +0.658579 0.759749 0.331526 0.0442155 +0.498896 0.823846 0.859371 0.0367959 +0.856591 0.905481 0.277116 0.0263332 +0.185676 0.454064 0.6736 0.0314353 +0.626468 0.0439294 0.443663 0.0362719 +0.864829 0.409669 0.890183 0.0541851 +0.393769 0.34907 0.796777 0.0511298 +0.566768 0.0917282 0.869829 0.0562265 +0.388211 0.212217 0.054938 0.0330852 +0.601323 0.924592 0.0856226 0.0378184 +0.762867 0.00295222 0.331161 0.0350963 +0.0894427 0.0139584 0.0626919 0.037032 +0.108992 0.262477 0.95901 0.0473186 +0.0197744 0.45409 0.252232 0.0415399 +0.322808 0.321967 0.0134081 0.0581444 +0.626393 0.844085 0.792714 0.044901 +0.0665854 0.459069 0.143628 0.0309899 +0.919309 0.49573 0.678941 0.037444 +0.266801 0.0587349 0.807871 0.061324 +0.872123 0.295287 0.320924 0.0703189 +0.0512992 0.39362 0.306433 0.0455702 +0.264077 0.692722 0.609153 0.0203218 +0.799182 0.0911076 0.902198 0.0390166 +0.559588 0.86543 0.146013 0.0564631 +0.603591 0.416666 0.866964 0.0310244 +0.78409 0.165627 0.911234 0.0375507 +0.0303547 0.237701 0.674878 0.0378285 +0.421948 0.534747 0.00733479 0.0264946 +0.341891 0.946361 0.839465 0.0640333 +0.434344 0.752409 0.854566 0.0563901 +0.614568 0.0504271 0.378391 0.0303934 +0.0499142 0.339405 0.60719 0.0352329 +0.878333 0.913093 0.638277 0.0385492 +0.870575 0.95751 0.859073 0.0446707 +0.531866 0.311198 0.0876051 0.0260822 +0.0101054 0.639709 0.403186 0.0442277 +0.293677 0.454796 0.240948 0.0534805 +0.521249 0.697468 0.370648 0.0239835 +0.859947 0.036666 0.485877 0.0292756 +0.678729 0.374071 0.579669 0.042083 +0.78147 0.0684407 0.548075 0.0350918 +0.126049 0.860938 0.0314799 0.0340261 +0.18011 0.795482 0.17652 0.0269563 +0.11824 0.0462406 0.123195 0.0373459 +0.656683 0.348767 0.320605 0.050579 +0.0187571 0.896383 0.166383 0.0288337 +0.424505 0.286134 0.479409 0.0435752 +0.994329 0.0449131 0.0250937 0.0160519 +0.708079 0.179294 0.385262 0.0360703 +0.762064 0.0682138 0.677143 0.0359194 +0.96755 0.564742 0.896969 0.0418576 +0.712103 0.318107 0.636889 0.0446342 +0.177279 0.316085 0.229138 0.0432381 +1.00784 0.485601 0.710132 0.0569629 +0.721448 -0.02603 0.81746 0.0413811 +0.532392 -0.016544 0.444982 0.038981 +0.558505 0.290145 1.03066 0.0359976 +-0.003511 0.454515 0.321203 0.03095 +0.207609 0.161252 -0.032115 0.0365915 +-0.02265 0.322714 0.44168 0.0556003 +0.112799 0.575721 -0.00735 0.0364405 +0.442789 0.252093 1.01168 0.0471642 +0.206534 1.00629 0.141621 0.045277 +1.00881 0.800926 0.075799 0.0357783 +0.0707515 0.460509 1.01662 0.0299879 +1.01544 0.767054 0.739801 0.0472356 +0.446789 -0.001768 0.845681 0.0325518 +0.474591 0.604961 1.02409 0.0307525 +0.794228 1.01113 0.453723 0.0481392 +1.03192 0.344468 0.831231 0.0432499 +0.685524 1.00707 0.963266 0.0287685 +0.0509262 -0.036883 0.62774 0.059891 +1.05093 0.963117 0.62774 0.059891 +1.05093 -0.036883 0.62774 0.059891 +0.109226 -0.02153 0.00297037 0.0351999 +0.109226 0.97847 1.00297 0.0351999 +0.109226 -0.02153 1.00297 0.0351999 +1.03154 0.798681 0.277237 0.0419151 +0.29497 -0.022211 0.496102 0.0292425 +0.838548 1.02317 0.539975 0.0304454 +0.497409 0.106686 -0.032474 0.0615782 +-0.024786 0.456821 0.485621 0.0485666 +0.663286 -0.038336 0.288148 0.0586649 +0.652296 1.02711 0.892742 0.0362347 +1.00109 0.649241 0.88736 0.0495611 +0.74619 -0.023172 0.0748496 0.0420883 +0.437479 0.670469 -0.022435 0.0577523 +0.448415 -0.014758 0.45695 0.0458634 +0.926334 1.00477 0.910503 0.0369942 +0.10132 -0.011253 0.457296 0.0483041 +0.122474 0.796274 1.02745 0.0308615 +0.324982 1.00117 0.761551 0.0327184 +0.813658 -0.006423 0.97337 0.0809183 +0.813658 0.993577 -0.02663 0.0809183 +0.813658 -0.006423 -0.02663 0.0809183 +0.775039 0.138755 1.00537 0.057073 +0.0410867 0.652444 -0.031948 0.0376408 +0.838553 0.812034 -0.029873 0.0370024 +0.940655 -0.000846 0.782406 0.0410872 +0.017358 0.245895 -0.038763 0.0458302 +1.01736 0.245895 0.961237 0.0458302 +1.01736 0.245895 -0.038763 0.0458302 +1.02281 0.00662286 0.0860634 0.0339621 +0.0228084 1.00662 0.0860634 0.0339621 +1.02281 1.00662 0.0860634 0.0339621 +1.01817 0.318043 0.376182 0.0217174 +0.321115 1.01837 0.304246 0.0724431 +0.946254 -0.012177 0.460383 0.0333158 +0.825213 1.02177 0.193165 0.0342723 +0.833007 0.546011 1.00994 0.0361181 +0.402583 0.00971101 -0.018555 0.0442304 +0.402583 1.00971 0.981445 0.0442304 +0.402583 1.00971 -0.018555 0.0442304 +0.727073 1.06358 0.260393 0.0647258 +0.782832 -0.031636 0.205276 0.0349768 +0.95252 0.666023 1.04381 0.0443656 +1.0186 0.372613 0.149242 0.0269084 +0.779623 0.308832 1.02656 0.0606179 +1.00735 0.163268 0.618201 0.0585136 +0.479672 0.368724 1.022 0.075594 +0.30011 1.06948 0.102352 0.0742681 +-0.005119 0.696791 0.800742 0.0480186 +0.343677 0.61186 1.01909 0.0444255 +0.63399 -0.01722 0.0938013 0.0294109 +1.00896 0.23089 0.830411 0.0414892 +0.882282 0.299867 1.01353 0.0432533 +0.672377 1.01606 0.641506 0.072129 +0.167655 0.929474 1.03131 0.0461509 +0.059528 0.272741 1.02365 0.0341338 +1.04908 0.396374 0.754566 0.0509116 +0.470229 1.06048 0.629549 0.0675032 +-0.005231 0.346538 0.213289 0.0424189 +1.01435 0.41772 0.39742 0.0555479 +0.226592 -0.068758 0.755676 0.0792953 +0.585488 1.00325 0.888589 0.0348285 +-2e-05 0.703688 0.442064 0.0313199 +0.676291 1.03622 0.0451658 0.0543215 +0.676291 0.036224 1.04517 0.0543215 +0.676291 1.03622 1.04517 0.0543215 +-0.040227 0.384112 0.793529 0.0472952 +0.0254624 0.157661 -0.008464 0.0416001 +1.02546 0.157661 0.991536 0.0416001 +1.02546 0.157661 -0.008464 0.0416001 +1.01216 0.337893 0.0185734 0.0465778 +0.012159 0.337893 1.01857 0.0465778 +1.01216 0.337893 1.01857 0.0465778 +0.268379 0.00918446 -0.007051 0.0546149 +0.268379 1.00918 0.992949 0.0546149 +0.268379 1.00918 -0.007051 0.0546149 +0.603378 -0.025122 0.487254 0.0440824 +0.282124 0.668662 1.02792 0.0347093 +-0.003504 0.074498 0.256737 0.0801265 +0.996496 1.0745 0.256737 0.0801265 +-0.003504 1.0745 0.256737 0.0801265 +1.02594 0.824566 0.787878 0.0284572 +0.354784 -0.046361 0.162629 0.0672973 +0.63117 -0.032264 0.939447 0.042208 +-0.01098 0.686564 0.674521 0.0561929 +0.406939 0.586467 -0.01528 0.0319148 +1.01759 0.422564 0.0729844 0.054215 +-0.019375 0.261192 0.35711 0.0490317 +-0.000159 0.548597 0.0212119 0.083188 +0.999841 0.548597 1.02121 0.083188 +-0.000159 0.548597 1.02121 0.083188 +1.02642 0.111922 0.847742 0.039674 +0.271135 1.00921 0.193371 0.0370391 +0.946594 0.311012 -0.025256 0.0326661 +0.899537 0.628654 -0.050909 0.064025 +-0.044359 0.119577 0.419929 0.0683606 +0.177981 1.02684 0.0494086 0.0534185 +0.177981 0.0268394 1.04941 0.0534185 +0.177981 1.02684 1.04941 0.0534185 +-0.021395 0.66228 0.297902 0.0408348 +1.00625 0.480994 0.834724 0.0677248 +0.200406 0.545933 -0.04436 0.0464773 +1.05021 0.579737 0.765942 0.0603907 +0.869145 0.883358 1.00873 0.0474228 +1.00231 0.635247 0.501495 0.0506369 +0.140248 0.188488 -0.013256 0.0376553 +0.288582 0.515884 1.00793 0.0603501 +1.01233 0.279954 0.617304 0.0358262 +0.120644 0.0894118 -0.032059 0.0642239 +0.289184 0.197058 1.02772 0.0707264 +1.01357 0.270546 0.229824 0.0375919 +-0.03156 0.485293 0.122876 0.0398037 +0.434766 1.02976 0.0860781 0.0670616 +0.118763 0.377762 -0.022586 0.0683615 +1.00786 0.115542 0.493049 0.0215798 +1.01157 0.287799 0.887511 0.0391695 +-0.037463 0.234295 0.134342 0.0391978 +0.7143 0.23185 -0.050088 0.0661416 +1.03374 0.415446 0.641129 0.0403581 +0.212817 -0.0617 0.473832 0.0645895 +0.41125 1.00308 0.236885 0.0270674 +0.152078 -0.003707 0.949202 0.0358281 +1.01303 0.0342143 0.711944 0.056647 +0.013031 1.03421 0.711944 0.056647 +1.01303 1.03421 0.711944 0.056647 +0.00694257 -0.005917 0.532325 0.0496413 +1.00694 0.994083 0.532325 0.0496413 +1.00694 -0.005917 0.532325 0.0496413 +1.00098 0.0577834 0.136637 0.0412142 +0.605473 0.409348 -0.032503 0.0673957 +-0.020441 0.22529 0.198031 0.027339 +-0.045849 0.594437 0.713845 0.0498706 +-0.022359 0.17473 0.325719 0.043001 +0.125902 1.022 0.706209 0.0567656 +1.04214 0.192512 0.265262 0.0466917 +0.8603 1.02866 0.878763 0.0298667 +1.02161 0.767948 0.816365 0.0295823 +-0.023368 0.304867 0.817437 0.0248115 +0.317356 0.763227 -0.025263 0.0366617 +-0.027667 0.625624 0.222697 0.0430629 +0.6258 1.00391 0.555842 0.0261327 +0.238154 1.04483 0.69253 0.0511755 +-0.031631 0.795145 0.562045 0.0421101 +0.692916 0.936996 1.006 0.0536423 +0.0612987 0.828561 1.03131 0.0383678 +0.27476 0.416635 1.00534 0.03989 +0.151219 0.269376 1.06409 0.0661351 +0.510268 0.264806 -9.5e-05 0.0225046 +-0.012316 0.794956 0.459355 0.062381 +0.337742 1.02315 0.938037 0.0349487 +0.0342908 -0.018703 0.366577 0.0412156 +1.03429 0.981297 0.366577 0.0412156 +1.03429 -0.018703 0.366577 0.0412156 +-0.014948 0.73436 0.266048 0.0382325 +0.425585 1.0059 0.746555 0.0691061 +-0.013824 0.0776001 0.940996 0.060906 +0.986176 0.0776001 -0.059004 0.060906 +-0.013824 0.0776001 -0.059004 0.060906 +0.904255 -0.033683 0.221476 0.066349 +-0.047119 0.707576 0.547516 0.0479972 +0.111766 0.720886 -0.022256 0.0600722 +0.635956 1.0298 0.794651 0.0632447 +-0.008351 0.880697 0.523276 0.044639 +-0.023219 0.197551 0.513835 0.0555144 +0.940707 1.02765 0.598567 0.0429915 +0.584229 -0.024785 0.0172434 0.0489605 +0.584229 0.975215 1.01724 0.0489605 +0.584229 -0.024785 1.01724 0.0489605 +0.208582 0.438481 -0.034275 0.0402734 +1.0155 0.58286 0.851049 0.0267722 +1.01481 0.937305 0.447386 0.0528306 +0.880042 1.00624 0.0687681 0.0359922 +-0.001453 0.721253 0.108041 0.0507827 +0.380704 0.108258 1.00344 0.0590851 +0.867518 -0.02834 0.584294 0.0418326 +0.211006 -0.015933 0.879521 0.0562441 +0.570928 1.0709 0.273395 0.0851397 +1.03465 0.841985 0.137435 0.0348042 +0.567662 1.02178 0.591851 0.043904 +-0.03926 0.808602 0.818023 0.0436351 +-0.000558 0.947992 0.75609 0.041168 +0.205006 0.403058 1.02419 0.0281731 +0.77182 0.555093 -0.042266 0.0448276 +0.430092 0.458727 -0.031043 0.0373765 +0.751363 1.0344 0.14959 0.0523993 +0.162644 0.491272 1.05762 0.075236 +0.688626 0.33076 -0.013667 0.0412615 +0.215031 1.01496 0.58225 0.0651206 +1.03029 0.687078 0.0328483 0.036585 +0.030286 0.687078 1.03285 0.036585 +1.03029 0.687078 1.03285 0.036585 +-0.003927 0.222829 0.0342757 0.0336669 +0.710889 -0.028597 0.903483 0.0453249 +1.00245 0.797205 0.656727 0.0420907 +0.742346 0.622211 1.04649 0.0702879 +-0.016309 0.918068 0.86768 0.0754331 +0.604434 0.0637653 -0.030706 0.053746 +0.611215 0.290741 -0.047103 0.0520755 +0.415433 1.00757 0.542266 0.048291 +1.08434 0.558346 0.209766 0.0882338 +0.942038 1.0111 0.106645 0.0368213 +1.02376 0.166046 0.160141 0.055723 +-0.013367 0.620803 0.814805 0.0296985 +0.608647 0.884135 1.00681 0.0458381 +0.424418 -0.042761 0.911392 0.0459779 +1.01614 0.608757 0.681981 0.0202771 +0.163826 1.02968 0.801277 0.0458751 +0.0654177 0.891432 -0.035887 0.0537495 +0.907726 1.02325 0.674226 0.0290572 +-0.029769 0.235422 0.435392 0.0318371 +0.636332 -0.023109 0.393936 0.0477772 +1.03941 0.289406 0.529162 0.056709 +0.77524 -0.016005 0.581261 0.0513153 +0.922219 0.395047 1.01646 0.0600072 +0.71157 0.410418 -0.005851 0.0420032 +1.03153 0.719653 0.51169 0.0392634 +1.01144 0.281378 0.756491 0.0480615 +0.34151 0.690674 -0.013162 0.0407582 +-0.048498 0.934567 0.590165 0.0501649 +0.365294 0.459056 1.01218 0.0342632 +0.0268705 0.440835 -0.036656 0.0417804 +1.02687 0.440835 0.963344 0.0417804 +1.02687 0.440835 -0.036656 0.0417804 +1.02594 0.530341 0.914529 0.0281468 +-0.021623 0.703441 0.194623 0.0398837 +1.00364 0.92026 0.233324 0.0438282 +1.01583 0.173466 0.905644 0.0454873 +-0.013132 0.483968 0.605124 0.0501303 +0.229132 0.355603 -0.034187 0.0448059 +0.909908 0.0463438 -0.022085 0.0289417 +1.07878 0.205243 0.422593 0.081558 +0.492898 -0.003958 0.992896 0.0478275 +0.492898 0.996042 -0.007104 0.0478275 +0.492898 -0.003958 -0.007104 0.0478275 +0.811262 0.59727 -0.012671 0.0177226 +0.521655 0.840888 1.02252 0.0525739 +0.744209 0.523271 1.01939 0.0298499 +0.504579 1.04841 0.514794 0.052888 +0.0441994 1.03683 0.00862465 0.0370856 +0.0441994 0.036832 1.00862 0.0370856 +0.0441994 1.03683 1.00862 0.0370856 +-0.055576 0.531099 0.275253 0.0686327 +0.487402 -0.015012 0.387992 0.0329924 +0.822029 -0.011078 0.744158 0.0839717 +1.02066 0.156694 0.746781 0.0709193 +0.172883 1.01405 0.423063 0.0349631 +0.502975 -0.056059 0.56179 0.0616785 +0.474474 -0.006188 0.196474 0.0485386 +0.579664 0.675727 1.03737 0.096589 +0.111246 -0.025669 0.213411 0.0782339 +1.0002 0.86589 0.037123 0.0334521 +-0.026069 0.309386 0.282966 0.0392466 +0.181608 0.301697 -0.025623 0.0275538 +0.199739 0.236496 -0.023818 0.0395169 +0.121279 1.0257 0.880398 0.0426744 +0.850605 0.468533 -0.024469 0.0504654 +-0.028237 0.380131 0.565324 0.0620944 +0.346012 1.01206 0.433007 0.0588545 +0.565364 -0.004182 0.397279 0.0203107 +0.554182 -0.051462 0.33933 0.0548596 +1.01051 0.550281 0.392836 0.0443045 +-0.019772 0.175531 0.230054 0.0256296 +0.385325 0.743079 1.02696 0.0348164 +-0.001578 0.970225 0.155892 0.0484727 +0.998422 -0.029775 0.155892 0.0484727 +-0.001578 -0.029775 0.155892 0.0484727 +-0.012365 0.599834 0.337308 0.0335558 +0.777571 0.871207 -0.027536 0.0466652 +0.406147 1.02571 0.884308 0.0298903 +0.455079 1.00211 0.305063 0.0532624 +0.287142 1.01668 0.89965 0.0280373 +0.248549 1.02349 0.397167 0.0455672 +-0.040752 0.563476 0.159068 0.0468383 +0.666744 0.136544 1.01286 0.0515035 +-0.087066 0.131675 0.0659722 0.0937147 +0.912934 0.131675 1.06597 0.0937147 +-0.087066 0.131675 1.06597 0.0937147 +0.527993 -0.014408 0.815854 0.0541892 +0.875825 -0.044247 0.426259 0.051261 +-0.019076 0.343436 0.677114 0.0559204 +0.0595963 -0.000819 0.303805 0.0287881 +0.833141 1.03667 0.270819 0.0451952 +0.188026 0.618244 1.02911 0.0573495 +0.227273 0.102636 1.01727 0.042538 +0.376566 0.395933 -0.034792 0.0452198 +-0.028464 0.375338 0.909365 0.0595384 +0.795194 -0.026502 0.863279 0.0325018 +0.423854 1.00572 0.38513 0.0327531 +1.01404 0.627803 0.120722 0.0447875 +0.672093 -0.019196 0.182765 0.0488043 +0.760403 0.472483 -0.027458 0.039872 +0.137313 -0.056031 0.355065 0.0689639 +0.0700236 -0.007533 0.804084 0.0550774 +0.464361 0.760942 1.03662 0.0467878 +-0.075107 0.864681 0.120223 0.0785918 +-0.013162 0.490436 0.191997 0.0319083 +0.65692 0.270291 1.0248 0.0355456 +-0.023722 0.888327 0.682703 0.0562087 +0.348935 -0.019239 0.604062 0.0463632 +0.698145 1.01817 0.50684 0.0611572 +0.254493 1.03093 0.488312 0.0380138 +0.827424 -0.042547 0.126521 0.0561172 +-0.005824 0.857331 0.599095 0.0347386 +-0.038587 0.962529 0.00876053 0.0741558 +0.961413 -0.037471 0.00876053 0.0741558 +0.961413 0.962529 1.00876 0.0741558 +-0.038587 -0.037471 0.00876053 0.0741558 +-0.038587 0.962529 1.00876 0.0741558 +0.961413 -0.037471 1.00876 0.0741558 +-0.038587 -0.037471 1.00876 0.0741558 +1.01909 0.0629969 0.596628 0.043016 +0.942486 1.01061 0.363133 0.0552184 +-0.011085 0.036028 0.843536 0.0450853 +0.988915 1.03603 0.843536 0.0450853 +-0.011085 1.03603 0.843536 0.0450853 +0.0578602 -0.009448 0.936978 0.0492956 +-0.004936 0.795563 0.196148 0.0457698 +0.809661 0.687451 1.00498 0.0322316 +-0.032768 0.772533 0.955006 0.0950878 +0.967232 0.772533 -0.044994 0.0950878 +-0.032768 0.772533 -0.044994 0.0950878 +-0.048936 0.551735 0.534833 0.0528596 +0.375699 0.875683 1.01893 0.0975151 +0.566335 -0.010361 0.149102 0.054902 +-0.015366 0.127304 0.550614 0.0213456 +1.00097 0.430829 0.163633 0.0355991 +0.668195 0.793406 1.00873 0.039842 +0.936557 0.240319 -0.009082 0.04043 +1.01466 0.739855 0.356655 0.0572485 +-0.027825 0.0718838 0.778897 0.0303641 +0.548701 -0.048761 0.691724 0.0762603 +0.258175 0.924392 1.00011 0.0310885 +0.82491 1.05147 0.367683 0.051718 +-0.034132 0.168922 0.839911 0.0345864 +1.00468 0.0284694 0.448191 0.0388986 +0.00468448 1.02847 0.448191 0.0388986 +1.00468 1.02847 0.448191 0.0388986 +0.270945 0.604011 1.00745 0.0295261 +-0.02749 0.347693 0.351578 0.0380249 +1.02453 0.601557 0.604077 0.0583827 +1.02825 0.738815 0.595255 0.0465328 +0.848475 0.760482 1.01277 0.030632 +0.855566 -0.036799 0.327218 0.0501047 +-0.0225 0.0755194 0.523888 0.0373642 +0.518427 0.511741 1.01302 0.0728522 +-0.043583 0.874734 0.341891 0.0830158 +0.738857 0.748209 -0.031146 0.0530326 +0.840014 0.364516 -0.031208 0.0398048 +0.654462 0.537034 -0.021659 0.0697949 +0.22809 0.697242 -0.027274 0.0476506 +0.843932 0.22086 -0.037848 0.0584928 +0.207324 -0.019648 0.665394 0.0252695 +0.918005 1.01555 0.520804 0.0389165 +0.235086 0.291119 1.0122 0.0348512 +0.605795 0.198742 -0.023806 0.0429829 +0.494755 1.0212 0.902942 0.0455957 +1.00322 0.49773 0.536065 0.0221596 +0.211255 0.827981 -0.045463 0.0854165 +0.101848 1.02665 0.550516 0.052328 +1.0413 0.293373 0.121345 0.0601096 +0.738643 0.810195 1.02808 0.0326969 +0.762867 1.00295 0.331161 0.0350963 +0.0894427 1.01396 0.0626919 0.037032 +0.108992 0.262477 -0.04099 0.0473186 +1.01977 0.45409 0.252232 0.0415399 +0.322808 0.321967 1.01341 0.0581444 +0.266801 1.05873 0.807871 0.061324 +1.03035 0.237701 0.674878 0.0378285 +0.421948 0.534747 1.00733 0.0264946 +0.341891 -0.053639 0.839465 0.0640333 +0.870575 -0.04249 0.859073 0.0446707 +1.01011 0.639709 0.403186 0.0442277 +0.126049 0.860938 1.03148 0.0340261 +1.01876 0.896383 0.166383 0.0288337 +-0.005671 0.0449131 0.0250937 0.0160519 +-0.03245 0.564742 0.896969 0.0418576 diff --git a/example/systems/summit/test-1200g.log b/example/systems/summit/test-1200g.log index fb6eeb2a..5f6788cc 100644 --- a/example/systems/summit/test-1200g.log +++ b/example/systems/summit/test-1200g.log @@ -1,4 +1,4 @@ -Fri Mar 16 11:58:54 EDT 2018 +Fri Mar 16 12:44:21 EDT 2018 ******************************************************** Running Unit Test for D3Q19 MPI Communication ******************************************************** @@ -18,11 +18,11 @@ Setting the distributions, size = : 11795503 ******************************************************** No. of timesteps for timing: 100 ******************************************************** -CPU time = 1.354868 -Lattice update rate (per process)= -705.856629 MLUPS -Lattice update rate (process)= -847027.954450 MLUPS +CPU time = 1.330030 +Lattice update rate (per process)= 886.860134 MLUPS +Lattice update rate (process)= 1064232.160581 MLUPS ******************************************************** -DRAM bandwidth (per process)= 943.718400 GB/sec +DRAM bandwidth (per process)= 339.710486 GB/sec Communication bandwidth (per process)= 2.813606 Gbit/sec Aggregated communication bandwidth = 3376.327680 Gbit/sec ****************************************** @@ -32,19 +32,59 @@ dist = 1.14 n= 1 ------------------------------------------------------------ -Sender: LSF System -Subject: Job 43312: in cluster Exited +Sender: LSF System +Subject: Job 43347: in cluster Exited -Job was submitted from host by user in cluster at Fri Mar 16 11:58:33 2018 -Job was executed on host(s) <1*batch1>, in queue , as user in cluster at Fri Mar 16 11:58:35 2018 - <42*a20n15> - <42*a20n16> - <42*a20n17> - <42*a20n18> - <42*a21n01> - <42*a21n02> - <42*a21n03> - <42*a21n04> +Job was submitted from host by user in cluster at Fri Mar 16 12:43:59 2018 +Job was executed on host(s) <1*batch2>, in queue , as user in cluster at Fri Mar 16 12:44:01 2018 + <42*a02n03> + <42*a02n04> + <42*a02n06> + <42*a02n07> + <42*a02n08> + <42*a02n09> + <42*a02n10> + <42*a02n18> + <42*a26n01> + <42*a26n02> + <42*a26n03> + <42*a26n04> + <42*a26n05> + <42*a26n06> + <42*a26n07> + <42*a26n08> + <42*a26n09> + <42*a26n10> + <42*a26n11> + <42*a26n12> + <42*a26n13> + <42*a26n14> + <42*a26n15> + <42*a26n16> + <42*a26n17> + <42*a26n18> + <42*a27n01> + <42*a27n02> + <42*a27n03> + <42*a27n04> + <42*a27n05> + <42*a27n06> + <42*a27n07> + <42*a27n08> + <42*a27n09> + <42*a27n10> + <42*a27n11> + <42*a27n13> + <42*a27n14> + <42*a27n15> + <42*a27n17> + <42*a27n18> + <42*a28n01> + <42*a28n02> + <42*a28n03> + <42*a28n04> + <42*a28n05> + <42*a28n06> <42*a28n07> <42*a28n08> <42*a28n09> @@ -197,51 +237,11 @@ Job was executed on host(s) <1*batch1>, in queue , as user in <42*a36n17> <42*a36n18> <42*b01n01> - <42*b01n02> - <42*b01n03> - <42*b01n04> - <42*b01n05> - <42*b01n06> - <42*b01n07> - <42*b01n08> - <42*b01n09> - <42*b01n10> - <42*b01n11> - <42*b01n12> - <42*b01n13> - <42*b01n14> - <42*b01n15> - <42*b01n16> - <42*b01n17> - <42*b01n18> - <42*b02n01> - <42*b02n02> - <42*b02n03> - <42*b02n04> - <42*b02n05> - <42*b02n06> - <42*b02n07> - <42*b02n09> - <42*b02n10> - <42*b02n11> - <42*b02n12> - <42*b02n13> - <42*b02n14> - <42*b02n15> - <42*b02n16> - <42*b02n17> - <42*b02n18> - <42*b03n01> - <42*b03n02> - <42*b03n03> - <42*b03n04> - <42*b03n05> - <42*b03n06> was used as the home directory. was used as the working directory. -Started at Fri Mar 16 11:58:35 2018 -Terminated at Fri Mar 16 11:59:26 2018 -Results reported at Fri Mar 16 11:59:26 2018 +Started at Fri Mar 16 12:44:01 2018 +Terminated at Fri Mar 16 12:44:49 2018 +Results reported at Fri Mar 16 12:44:49 2018 The output (if any) is above this job summary. diff --git a/example/systems/summit/test-3600g.log b/example/systems/summit/test-3600g.log index 7a6f40a5..6fd089c9 100644 --- a/example/systems/summit/test-3600g.log +++ b/example/systems/summit/test-3600g.log @@ -1,142 +1,60994 @@ -Fri Mar 16 11:29:36 EDT 2018 +Fri Mar 16 12:46:20 EDT 2018 ******************************************************** Running Unit Test for D3Q19 MPI Communication ******************************************************** ******************************************************** Sub-domain size = 320 x 320 x 320 -Parallel domain size = 10 x 10 x 6 +Parallel domain size = 15 x 15 x 16 ******************************************************** Assigning phase ID from file Initialize from segmented data: solid=0, NWP=1, WP=2 Media porosity = 0.359970 Domain set. -Create ScaLBL_Communicator -Set up memory efficient layout -Allocating distributions -Setting up device map and neighbor list -Setting the distributions, size = : 11795503 -******************************************************** -No. of timesteps for timing: 100 -******************************************************** -CPU time = 1.329585 -Lattice update rate (per process)= -719.278687 MLUPS -Lattice update rate (process)= -431567.212128 MLUPS -******************************************************** -DRAM bandwidth (per process)= 943.718400 GB/sec -Communication bandwidth (per process)= 2.813606 Gbit/sec -Aggregated communication bandwidth = 1688.163840 Gbit/sec +[a36n17:150929] *** Process received signal *** +[a36n17:150929] Signal: Segmentation fault (11) +[a36n17:150929] Signal code: (3) +[a36n17:150929] Failing at address: 0x5170000000000a0 +[a36n17:150929] [ 0] [0x1000000504d8] +[a36n17:150929] [ 1] [0x3ffff5e7c950] +[a36n17:150929] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n17:150929] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n17:150929] [ 4] [a34n12:145501] *** Process received signal *** +[a34n12:145501] Signal: Segmentation fault (11) +[a34n12:145501] Signal code: (3) +[a34n12:145501] Failing at address: 0x14170000000000a0 +[a34n12:145501] [ 0] [0x1000000504d8] +[a34n12:145501] [ 1] [0x3ffffb02a7a0] +[a34n12:145501] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n12:145501] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n12:145501] [ 4] [b15n08:118555] *** Process received signal *** +[b15n08:118555] Signal: Segmentation fault (11) +[b15n08:118555] Signal code: (3) +[b15n08:118555] Failing at address: 0xb170000000000a0 +[b15n08:118555] [ 0] [0x1000000504d8] +[b15n08:118555] [ 1] [0x3ffff6a27cc0] +[b15n08:118555] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n08:118555] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n08:118555] [ 4] [b17n11:124447] *** Process received signal *** +[b17n11:124447] Signal: Segmentation fault (11) +[b17n11:124447] Signal code: (3) +[b17n11:124447] Failing at address: 0xb170000000000a0 +[b17n11:124447] [ 0] [0x1000000504d8] +[b17n11:124447] [ 1] [0x3fffc55acff0] +[b17n11:124447] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n11:124447] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n11:124447] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n17:150929] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n17:150929] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n17:150929] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n17:150929] [ 8] [a35n16:148190] *** Process received signal *** +[a35n16:148190] Signal: Segmentation fault (11) +[a35n16:148190] Signal code: (3) +[a35n16:148190] Failing at address: 0x14170000000000a0 +[a35n16:148190] [ 0] [0x1000000504d8] +[a35n16:148190] [ 1] [0x3fffd3928e50] +[a35n16:148190] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n16:148190] [ 3] [b15n12:132101] *** Process received signal *** +[b15n12:132101] Signal: Segmentation fault (11) +[b15n12:132101] Signal code: (3) +[b15n12:132101] Failing at address: 0xb170000000000a0 +[b15n12:132101] [ 0] [0x1000000504d8] +[b15n12:132101] [ 1] [0x3fffc18fdcb0] +[b15n12:132101] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n12:132101] [ 3] [b22n01:122017] *** Process received signal *** +[b22n01:122017] Signal: Segmentation fault (11) +[b22n01:122017] Signal code: (3) +[b22n01:122017] Failing at address: 0x14170000000000a0 +[b22n01:122017] [ 0] [0x1000000504d8] +[b22n01:122017] [ 1] [0x3ffffa12cf50] +[b22n01:122017] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n01:122017] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n01:122017] [ 4] [b18n11:128955] *** Process received signal *** +[b18n11:128955] Signal: Segmentation fault (11) +[b18n11:128955] Signal code: (3) +[b18n11:128955] Failing at address: 0x14170000000000a0 +[b18n11:128955] [ 0] [0x1000000504d8] +[b18n11:128955] [ 1] [0x3fffed3d2c60] +[b18n11:128955] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n11:128955] [ 3] [b22n04:44757] *** Process received signal *** +[b22n04:44757] Signal: Segmentation fault (11) +[b22n04:44757] Signal code: (3) +[b22n04:44757] Failing at address: 0x14170000000000a0 +[b22n04:44757] [ 0] [0x1000000504d8] +[b22n04:44757] [ 1] [0x3fffdff837a0] +[b22n04:44757] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n04:44757] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n04:44757] [ 4] [b15n07:124353] *** Process received signal *** +[b15n07:124353] Signal: Segmentation fault (11) +[b15n07:124353] Signal code: (3) +[b15n07:124353] Failing at address: 0xb170000000000a0 +[b15n07:124353] [ 0] [0x1000000504d8] +[b15n07:124353] [ 1] [0x3fffdbc71f20] +[b15n07:124353] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n07:124353] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n07:124353] [ 4] [b11n05:162567] *** Process received signal *** +[b11n05:162567] Signal: Segmentation fault (11) +[b11n05:162567] Signal code: (3) +[b11n05:162567] Failing at address: 0x14170000000000a0 +[b11n05:162567] [ 0] [0x1000000504d8] +[b11n05:162567] [ 1] [0x3fffc2c1b240] +[b11n05:162567] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n05:162567] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n05:162567] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n12:145501] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n12:145501] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n12:145501] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n12:145501] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n12:145501] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n12:145501] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n12:145501] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n12:145501] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n12:145501] [13] [a31n14:165666] *** Process received signal *** +[a31n14:165666] Signal: Segmentation fault (11) +[a31n14:165666] Signal code: (3) +[a31n14:165666] Failing at address: 0x14170000000000a0 +[a31n14:165666] [ 0] [0x1000000504d8] +[a31n14:165666] [ 1] [0x3fffdf8a0000] +[a31n14:165666] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n14:165666] [ 3] [a28n09:03931] *** Process received signal *** +[a28n09:03931] Signal: Segmentation fault (11) +[a28n09:03931] Signal code: (3) +[a28n09:03931] Failing at address: 0x14170000000000a0 +[a28n09:03931] [ 0] [0x1000000504d8] +[a28n09:03931] [ 1] [0x3fffe4f608a0] +[a28n09:03931] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n09:03931] [ 3] [b13n17:129931] *** Process received signal *** +[b13n17:129931] Signal: Segmentation fault (11) +[b13n17:129931] Signal code: (3) +[b13n17:129931] Failing at address: 0xb170000000000a0 +[b13n17:129931] [ 0] [0x1000000504d8] +[b13n17:129931] [ 1] [0x3ffffc127940] +[b13n17:129931] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n17:129931] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n17:129931] [ 4] [b12n10:140993] *** Process received signal *** +[b12n10:140993] Signal: Segmentation fault (11) +[b12n10:140993] Signal code: (3) +[b12n10:140993] Failing at address: 0xb170000000000a0 +[b12n10:140993] [ 0] [0x1000000504d8] +[b12n10:140993] [ 1] [0x3ffffa7d5f60] +[b12n10:140993] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n10:140993] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n10:140993] [ 4] [b18n15:120134] *** Process received signal *** +[b18n15:120134] Signal: Segmentation fault (11) +[b18n15:120134] Signal code: (3) +[b18n15:120134] Failing at address: 0x14170000000000a0 +[b18n15:120134] [ 0] [0x1000000504d8] +[b18n15:120134] [ 1] [0x3fffd40d3620] +[b18n15:120134] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n15:120134] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n15:120134] [ 4] [b17n07:125475] *** Process received signal *** +[b17n07:125475] Signal: Segmentation fault (11) +[b17n07:125475] Signal code: (3) +[b17n07:125475] Failing at address: 0x5170000000000a0 +[b17n07:125475] [ 0] [0x1000000504d8] +[b17n07:125475] [ 1] [0x3fffe8c9f7d0] +[b17n07:125475] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n07:125475] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n07:125475] [ 4] [a32n10:162612] *** Process received signal *** +[a32n10:162612] Signal: Segmentation fault (11) +[a32n10:162612] Signal code: (3) +[a32n10:162612] Failing at address: 0x14170000000000a0 +[a32n10:162612] [ 0] [0x1000000504d8] +[a32n10:162612] [ 1] [0x3fffcca82560] +[a32n10:162612] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n10:162612] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n10:162612] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n17:150929] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n17:150929] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n17:150929] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n17:150929] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n16:148190] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n16:148190] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n16:148190] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n16:148190] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n16:148190] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n16:148190] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n16:148190] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n16:148190] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n16:148190] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n16:148190] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n16:148190] [14] [a31n03:158657] *** Process received signal *** +[a31n03:158657] Signal: Segmentation fault (11) +[a31n03:158657] Signal code: (3) +[a31n03:158657] Failing at address: 0xb170000000000a0 +[a31n03:158657] [ 0] [0x1000000504d8] +[a31n03:158657] [ 1] [0x3fffe95feb10] +[a31n03:158657] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n03:158657] [ 3] [b14n01:128006] *** Process received signal *** +[b14n01:128006] Signal: Segmentation fault (11) +[b14n01:128006] Signal code: (3) +[b14n01:128006] Failing at address: 0xb170000000000a0 +[b14n01:128006] [ 0] [0x1000000504d8] +[b14n01:128006] [ 1] [0x3fffdd411a00] +[b14n01:128006] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n01:128006] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n01:128006] [ 4] [b05n07:08399] *** Process received signal *** +[b05n07:08399] Signal: Segmentation fault (11) +[b05n07:08399] Signal code: (3) +[b05n07:08399] Failing at address: 0x14170000000000a0 +[b05n07:08399] [ 0] [0x1000000504d8] +[b05n07:08399] [ 1] [0x3fffe2350040] +[b05n07:08399] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n07:08399] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n01:122017] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n01:122017] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n01:122017] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n01:122017] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n01:122017] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n01:122017] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n01:122017] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n01:122017] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n01:122017] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n01:122017] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n01:122017] *** End of error message *** +[b02n04:168492] *** Process received signal *** +[b02n04:168492] Signal: Segmentation fault (11) +[b02n04:168492] Signal code: (3) +[b02n04:168492] Failing at address: 0x14170000000000a0 +[b02n04:168492] [ 0] [0x1000000504d8] +[b02n04:168492] [ 1] [0x3fffc40c79e0] +[b02n04:168492] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n04:168492] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n04:44757] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n04:44757] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n04:44757] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n04:44757] [ 8] [b18n18:121305] *** Process received signal *** +[b18n18:121305] Signal: Segmentation fault (11) +[b18n18:121305] Signal code: (3) +[b18n18:121305] Failing at address: 0x14170000000000a0 +[b18n18:121305] [ 0] [0x1000000504d8] +[b18n18:121305] [ 1] [0x3fffe1c5ef90] +[b18n18:121305] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n18:121305] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n18:121305] [ 4] [b07n07:163255] *** Process received signal *** +[b07n07:163255] Signal: Segmentation fault (11) +[b07n07:163255] Signal code: (3) +[b07n07:163255] Failing at address: 0x14170000000000a0 +[b07n07:163255] [ 0] [0x1000000504d8] +[b07n07:163255] [ 1] [0x3fffdbc45ee0] +[b07n07:163255] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n07:163255] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n07:124353] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n07:124353] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n07:124353] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n07:124353] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n07:124353] [ 9] [a32n08:165283] *** Process received signal *** +[a32n08:165283] Signal: Segmentation fault (11) +[a32n08:165283] Signal code: (3) +[a32n08:165283] Failing at address: 0xb170000000000a0 +[a32n08:165283] [ 0] [0x1000000504d8] +[a32n08:165283] [ 1] [0x3ffff9165200] +[a32n08:165283] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n08:165283] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n08:165283] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n05:162567] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n05:162567] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n05:162567] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n05:162567] [ 8] [a31n05:158577] *** Process received signal *** +[a31n05:158577] Signal: Segmentation fault (11) +[a31n05:158577] Signal code: (3) +[a31n05:158577] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n09:03931] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n09:03931] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n09:03931] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n11:124447] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n11:124447] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n11:124447] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n11:124447] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n11:124447] [ 9] [b18n13:123056] *** Process received signal *** +[b18n13:123056] Signal: Segmentation fault (11) +[b18n13:123056] Signal code: (3) +[b18n13:123056] Failing at address: 0x14170000000000a0 +[b18n13:123056] [ 0] [0x1000000504d8] +[b18n13:123056] [ 1] [0x3fffc8624260] +[b18n13:123056] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n13:123056] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n13:123056] [ 4] [b03n04:04851] *** Process received signal *** +[b03n04:04851] Signal: Segmentation fault (11) +[b03n04:04851] Signal code: (3) +[b03n04:04851] Failing at address: 0xb170000000000a0 +[b03n04:04851] [ 0] [0x1000000504d8] +[b03n04:04851] [ 1] [0x3fffec18b940] +[b03n04:04851] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n04:04851] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n04:04851] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n07:125475] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n07:125475] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n07:125475] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n07:125475] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n07:125475] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n07:125475] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n07:125475] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n07:125475] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n17:150929] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n17:150929] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n17:150929] *** End of error message *** +[a34n13:137746] *** Process received signal *** +[a34n13:137746] Signal: Segmentation fault (11) +[a34n13:137746] Signal code: (3) +[a34n13:137746] Failing at address: 0xb170000000000a0 +[a34n13:137746] [ 0] [0x1000000504d8] +[a34n13:137746] [ 1] [0x3fffdfb3a5f0] +[a34n13:137746] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n13:137746] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n13:137746] [ 4] [a29n12:160192] *** Process received signal *** +[a29n12:160192] Signal: Segmentation fault (11) +[a29n12:160192] Signal code: (3) +[a29n12:160192] Failing at address: 0xb170000000000a0 +[a29n12:160192] [ 0] [0x1000000504d8] +[a29n12:160192] [ 1] [0x3fffe6a7cb40] +[a29n12:160192] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n12:160192] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n12:160192] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n16:148190] *** End of error message *** +[b15n09:121756] *** Process received signal *** +[b15n09:121756] Signal: Segmentation fault (11) +[b15n09:121756] Signal code: (3) +[b15n09:121756] Failing at address: 0x14170000000000a0 +[b15n09:121756] [ 0] [0x1000000504d8] +[b15n09:121756] [ 1] [0x3fffe2918f90] +[b15n09:121756] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n09:121756] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n09:121756] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n07:08399] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n07:08399] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n07:08399] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n04:168492] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n04:168492] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n04:168492] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n12:145501] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n12:145501] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n08:118555] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n08:118555] [ 6] [b21n13:117512] *** Process received signal *** +[b21n13:117512] Signal: Segmentation fault (11) +[b21n13:117512] Signal code: (3) +[b21n13:117512] Failing at address: 0xb170000000000a0 +[b21n13:117512] [ 0] [0x1000000504d8] +[b21n13:117512] [ 1] [0x3fffe2dfc090] +[b21n13:117512] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n13:117512] [ 3] [b12n12:138739] *** Process received signal *** +[b12n12:138739] Signal: Segmentation fault (11) +[b12n12:138739] Signal code: (3) +[b12n12:138739] Failing at address: 0x5170000000000a0 +[b12n12:138739] [ 0] [0x1000000504d8] +[b12n12:138739] [ 1] [0x3ffff64df0b0] +[b12n12:138739] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n12:138739] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n12:138739] [ 4] [b22n12:122965] *** Process received signal *** +[b22n12:122965] Signal: Segmentation fault (11) +[b22n12:122965] Signal code: (3) +[b22n12:122965] Failing at address: 0x14170000000000a0 +[b22n12:122965] [ 0] [0x1000000504d8] +[b22n12:122965] [ 1] [0x3fffd72c9350] +[b22n12:122965] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n12:122965] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n12:122965] [ 4] [b07n08:172901] *** Process received signal *** +[b07n08:172901] Signal: Segmentation fault (11) +[b07n08:172901] Signal code: (3) +[b07n08:172901] Failing at address: 0xb170000000000a0 +[b07n08:172901] [ 0] [0x1000000504d8] +[b07n08:172901] [ 1] [0x3fffc12f59f0] +[b07n08:172901] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n08:172901] [ 3] [b21n08:116173] *** Process received signal *** +[b21n08:116173] Signal: Segmentation fault (11) +[b21n08:116173] Signal code: (3) +[b21n08:116173] Failing at address: 0xb170000000000a0 +[b21n08:116173] [ 0] [0x1000000504d8] +[b21n08:116173] [ 1] [0x3fffc732ef60] +[b21n08:116173] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n08:116173] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n07:124353] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n07:124353] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n07:124353] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n07:124353] [13] [b20n04:126346] *** Process received signal *** +[b20n04:126346] Signal: Segmentation fault (11) +[b20n04:126346] Signal code: (3) +[b20n04:126346] Failing at address: 0xb170000000000a0 +[b20n04:126346] [ 0] [0x1000000504d8] +[b20n04:126346] [ 1] [0x3fffeee672f0] +[b20n04:126346] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n04:126346] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n09:03931] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n09:03931] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n09:03931] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n09:03931] [10] [a33n15:148581] *** Process received signal *** +[a33n15:148581] Signal: Segmentation fault (11) +[a33n15:148581] Signal code: (3) +[a33n15:148581] Failing at address: 0xb170000000000a0 +[a33n15:148581] [ 0] [0x1000000504d8] +[a33n15:148581] [ 1] [0x3fffccea3800] +[a33n15:148581] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n15:148581] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n15:148581] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n17:129931] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n17:129931] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n17:129931] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n17:129931] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n17:129931] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n17:129931] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n17:129931] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n12:132101] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n12:132101] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n12:132101] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n11:124447] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n11:124447] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n11:124447] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n10:140993] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n10:140993] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n10:140993] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n10:140993] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n04:04851] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n04:04851] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n04:04851] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n04:04851] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n04:04851] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n11:128955] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n11:128955] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n11:128955] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n11:128955] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n07:125475] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n07:125475] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n07:125475] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n04:44757] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n04:44757] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n04:44757] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n07:163255] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n07:163255] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n07:163255] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n07:163255] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n07:163255] [ 8] [a32n02:157955] *** Process received signal *** +[a32n02:157955] Signal: Segmentation fault (11) +[a32n02:157955] Signal code: (3) +[a32n02:157955] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n08:165283] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n08:165283] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n08:165283] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n08:165283] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n05:162567] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n05:162567] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n05:162567] [11] [a30n02:155381] *** Process received signal *** +[a30n02:155381] Signal: Segmentation fault (11) +[a30n02:155381] Signal code: (3) +[a30n02:155381] Failing at address: 0xb170000000000a0 +[a30n02:155381] [ 0] [0x1000000504d8] +[a30n02:155381] [ 1] [0x3fffd4c563d0] +[a30n02:155381] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n02:155381] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n02:155381] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n02:155381] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n02:155381] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n02:155381] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n10:162612] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n10:162612] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n10:162612] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n10:162612] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n10:162612] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n10:162612] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n10:162612] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n10:162612] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n10:162612] [13] [a35n18:145744] *** Process received signal *** +[a35n18:145744] Signal: Segmentation fault (11) +[a35n18:145744] Signal code: (3) +[a35n18:145744] Failing at address: 0xb170000000000a0 +[a35n18:145744] [ 0] [0x1000000504d8] +[a35n18:145744] [ 1] [0x3ffffd61de40] +[a35n18:145744] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n18:145744] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n18:145744] [ 4] [a33n12:168646] *** Process received signal *** +[a33n12:168646] Signal: Segmentation fault (11) +[a33n12:168646] Signal code: (3) +[a33n12:168646] Failing at address: 0xb170000000000a0 +[a33n12:168646] [ 0] [0x1000000504d8] +[a33n12:168646] [ 1] [0x3fffd1cd5740] +[a33n12:168646] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n12:168646] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n12:168646] [ 4] [b14n04:126862] *** Process received signal *** +[b14n04:126862] Signal: Segmentation fault (11) +[b14n04:126862] Signal code: (3) +[b14n04:126862] Failing at address: 0x14170000000000a0 +[b14n04:126862] [ 0] [0x1000000504d8] +[b14n04:126862] [ 1] [0x3ffff6aa2ad0] +[b14n04:126862] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n04:126862] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n04:126862] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n07:08399] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n07:08399] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n07:08399] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n07:08399] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n15:120134] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n15:120134] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n15:120134] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n15:120134] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n15:120134] [ 9] [a30n15:154387] *** Process received signal *** +[a30n15:154387] Signal: Segmentation fault (11) +[a30n15:154387] Signal code: (3) +[a30n15:154387] Failing at address: 0x14170000000000a0 +[a30n15:154387] [ 0] [0x1000000504d8] +[a30n15:154387] [ 1] [0x3fffd2770d40] +[a30n15:154387] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n15:154387] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n13:117512] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n13:117512] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n13:117512] [ 6] [b14n11:130808] *** Process received signal *** +[b14n11:130808] Signal: Segmentation fault (11) +[b14n11:130808] Signal code: (3) +[b14n11:130808] Failing at address: 0x14170000000000a0 +[b14n11:130808] [ 0] [0x1000000504d8] +[b14n11:130808] [ 1] [0x3fffeee3d780] +[b14n11:130808] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n11:130808] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n13:137746] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n13:137746] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n13:137746] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n13:137746] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n13:137746] [ 9] [b20n01:128832] *** Process received signal *** +[b20n01:128832] Signal: Segmentation fault (11) +[b20n01:128832] Signal code: (3) +[b20n01:128832] Failing at address: 0x14170000000000a0 +[b20n01:128832] [ 0] [0x1000000504d8] +[b20n01:128832] [ 1] [0x3fffd854c790] +[b20n01:128832] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n01:128832] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n01:128832] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n14:165666] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n14:165666] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n14:165666] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n14:165666] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n14:165666] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n14:165666] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n14:165666] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n14:165666] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n14:165666] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n14:165666] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n09:03931] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n09:03931] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n09:03931] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n09:03931] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n09:03931] *** End of error message *** +[a33n13:168163] *** Process received signal *** +[a33n13:168163] Signal: Segmentation fault (11) +[a33n13:168163] Signal code: (3) +[a33n13:168163] Failing at address: 0x14170000000000a0 +[a33n13:168163] [ 0] [0x1000000504d8] +[a33n13:168163] [ 1] [0x3fffd35fecc0] +[a33n13:168163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n13:168163] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n13:168163] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n04:04851] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n04:04851] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n04:04851] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n04:04851] [13] [b03n12:167679] *** Process received signal *** +[b03n12:167679] Signal: Segmentation fault (11) +[b03n12:167679] Signal code: (3) +[b03n12:167679] Failing at address: 0x14170000000000a0 +[b03n12:167679] [ 0] [0x1000000504d8] +[b03n12:167679] [ 1] [0x3fffd55f4670] +[b03n12:167679] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n12:167679] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n12:167679] [ 4] [b13n04:129408] *** Process received signal *** +[b13n04:129408] Signal: Segmentation fault (11) +[b13n04:129408] Signal code: (3) +[b13n04:129408] Failing at address: 0xb170000000000a0 +[b13n04:129408] [ 0] [0x1000000504d8] +[b13n04:129408] [ 1] [0x3fffe1a4bb70] +[b13n04:129408] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n04:129408] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n04:129408] [ 4] [a32n02:157955] [ 0] [0x1000000504d8] +[a32n02:157955] [ 1] [0x3fffe549db30] +[a32n02:157955] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n02:157955] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n02:157955] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n04:168492] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n04:168492] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n04:168492] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n04:168492] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n18:121305] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n18:121305] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n18:121305] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n18:121305] [ 8] [a30n13:154232] *** Process received signal *** +[a30n13:154232] Signal: Segmentation fault (11) +[a30n13:154232] Signal code: (3) +[a30n13:154232] Failing at address: 0xb170000000000a0 +[a30n13:154232] [ 0] [0x1000000504d8] +[a30n13:154232] [ 1] [0x3fffdd2d8390] +[a30n13:154232] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n13:154232] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n13:154232] [ 4] [b11n11:152989] *** Process received signal *** +[b11n11:152989] Signal: Segmentation fault (11) +[b11n11:152989] Signal code: (3) +[b11n11:152989] Failing at address: 0xb170000000000a0 +[b11n11:152989] [ 0] [0x1000000504d8] +[b11n11:152989] [ 1] [0x3fffcd0f9740] +[b11n11:152989] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n11:152989] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n11:152989] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n11:152989] [ 5] [a29n17:169739] *** Process received signal *** +[a29n17:169739] Signal: Segmentation fault (11) +[a29n17:169739] Signal code: (3) +[a29n17:169739] Failing at address: 0x14170000000000a0 +[a29n17:169739] [ 0] [0x1000000504d8] +[a29n17:169739] [ 1] [0x3fffec28ea70] +[a29n17:169739] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n17:169739] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n17:169739] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n03:158657] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n03:158657] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n03:158657] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n18:145744] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n18:145744] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n18:145744] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n18:145744] [ 8] [b09n05:173824] *** Process received signal *** +[b09n05:173824] Signal: Segmentation fault (11) +[b09n05:173824] Signal code: (3) +[b09n05:173824] Failing at address: 0xb170000000000a0 +[b09n05:173824] [ 0] [0x1000000504d8] +[b09n05:173824] [ 1] [0x3fffc845fc80] +[b09n05:173824] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n05:173824] [ 3] [a33n18:149118] *** Process received signal *** +[a33n18:149118] Signal: Segmentation fault (11) +[a33n18:149118] Signal code: (3) +[a33n18:149118] Failing at address: 0x14170000000000a0 +[a33n18:149118] [ 0] [0x1000000504d8] +[a33n18:149118] [ 1] [0x3fffca283df0] +[a33n18:149118] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n18:149118] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n18:149118] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n01:128006] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n01:128006] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n01:128006] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n01:128006] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n01:128006] [ 9] [b02n01:167299] *** Process received signal *** +[b02n01:167299] Signal: Segmentation fault (11) +[b02n01:167299] Signal code: (3) +[b02n01:167299] Failing at address: 0xb170000000000a0 +[b02n01:167299] [ 0] [0x1000000504d8] +[b02n01:167299] [ 1] [0x3fffe7eb9460] +[b02n01:167299] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n01:167299] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n01:167299] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n11:124447] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n11:124447] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n11:124447] *** End of error message *** +[b05n16:168051] *** Process received signal *** +[b05n16:168051] Signal: Segmentation fault (11) +[b05n16:168051] Signal code: (3) +[b05n16:168051] Failing at address: 0x5170000000000a0 +[b05n16:168051] [ 0] [0x1000000504d8] +[b05n16:168051] [ 1] [0x3fffee6e62a0] +[b05n16:168051] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n16:168051] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n16:168051] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n13:123056] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n13:123056] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n13:123056] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n13:123056] [ 8] [b21n16:116527] *** Process received signal *** +[b21n16:116527] Signal: Segmentation fault (11) +[b21n16:116527] Signal code: (3) +[b21n16:116527] Failing at address: 0x14170000000000a0 +[b21n16:116527] [ 0] [0x1000000504d8] +[b21n16:116527] [ 1] [0x3fffcedc5f50] +[b21n16:116527] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n16:116527] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n08:165283] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n08:165283] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n08:165283] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n08:165283] [12] [b14n07:127356] *** Process received signal *** +[b14n07:127356] Signal: Segmentation fault (11) +[b14n07:127356] Signal code: (3) +[b14n07:127356] Failing at address: 0xb170000000000a0 +[b14n07:127356] [ 0] [0x1000000504d8] +[b14n07:127356] [ 1] [0x3ffff8e06d20] +[b14n07:127356] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n07:127356] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n07:127356] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n07:127356] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n07:127356] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n07:127356] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n07:127356] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n13:137746] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n13:137746] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n13:137746] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n13:137746] [13] [a35n12:145264] *** Process received signal *** +[a35n12:145264] Signal: Segmentation fault (11) +[a35n12:145264] Signal code: (3) +[a35n12:145264] Failing at address: 0xb170000000000a0 +[a35n12:145264] [ 0] [0x1000000504d8] +[a35n12:145264] [ 1] [0x3fffc4173770] +[a35n12:145264] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n12:145264] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n12:145264] [ 4] [a29n08:159023] *** Process received signal *** +[a29n08:159023] Signal: Segmentation fault (11) +[a29n08:159023] Signal code: (3) +[a29n08:159023] Failing at address: 0xb170000000000a0 +[a29n08:159023] [ 0] [0x1000000504d8] +[a29n08:159023] [ 1] [0x3ffff9cc45b0] +[a29n08:159023] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n08:159023] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n09:121756] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n09:121756] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n09:121756] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n09:121756] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n12:138739] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n12:138739] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n12:138739] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n12:138739] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n12:138739] [ 9] [b08n15:177823] *** Process received signal *** +[b08n15:177823] Signal: Segmentation fault (11) +[b08n15:177823] Signal code: (3) +[b08n15:177823] Failing at address: 0x14170000000000a0 +[b08n15:177823] [ 0] [0x1000000504d8] +[b08n15:177823] [ 1] [0x3fffcabea1d0] +[b08n15:177823] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n15:177823] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n15:177823] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n15:177823] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n04:44757] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n04:44757] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n04:44757] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n04:44757] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n11:128955] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n11:128955] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n11:128955] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n12:122965] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n12:122965] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n12:122965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n12:122965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n12:122965] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n04:04851] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n04:04851] *** End of error message *** +[b03n15:85941] *** Process received signal *** +[b03n15:85941] Signal: Segmentation fault (11) +[b03n15:85941] Signal code: (3) +[b03n15:85941] Failing at address: 0xb170000000000a0 +[b03n15:85941] [ 0] [0x1000000504d8] +[b03n15:85941] [ 1] [0x3fffe1acc8b0] +[b03n15:85941] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n15:85941] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n15:85941] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n08:172901] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n08:172901] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n08:172901] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n02:157955] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n13:117512] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n13:117512] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n13:117512] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n13:117512] [10] [a26n15:55852] *** Process received signal *** +[a26n15:55852] Signal: Segmentation fault (11) +[a26n15:55852] Signal code: (3) +[a26n15:55852] Failing at address: 0xb170000000000a0 +[a26n15:55852] [ 0] [0x1000000504d8] +[a26n15:55852] [ 1] [0x3fffe80716d0] +[a26n15:55852] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n15:55852] [ 3] [a28n03:10293] *** Process received signal *** +[a28n03:10293] Signal: Segmentation fault (11) +[a28n03:10293] Signal code: (3) +[a28n03:10293] Failing at address: 0x14170000000000a0 +[a28n03:10293] [ 0] [0x1000000504d8] +[a28n03:10293] [ 1] [0x3fffd6e88a20] +[a28n03:10293] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n03:10293] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n03:10293] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n03:10293] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n03:10293] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n05:162567] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n05:162567] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n05:162567] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n05:162567] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n05:173824] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n05:173824] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n05:173824] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n05:173824] [ 7] [b10n18:162424] *** Process received signal *** +[b10n18:162424] Signal: Segmentation fault (11) +[b10n18:162424] Signal code: (3) +[b10n18:162424] Failing at address: 0xb170000000000a0 +[b10n18:162424] [ 0] [0x1000000504d8] +[b10n18:162424] [ 1] [0x3fffd0387490] +[b10n18:162424] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n18:162424] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n18:162424] [ 4] [b20n03:124683] *** Process received signal *** +[b20n03:124683] Signal: Segmentation fault (11) +[b20n03:124683] Signal code: (3) +[b20n03:124683] Failing at address: 0x14170000000000a0 +[b20n03:124683] [ 0] [0x1000000504d8] +[b20n03:124683] [ 1] [0x3fffc36694e0] +[b20n03:124683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n03:124683] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n03:124683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n16:168051] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n16:168051] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n16:168051] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n16:168051] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n16:168051] [ 9] [a31n05:158577] [ 0] [0x1000000504d8] +[a31n05:158577] [ 1] [0x3fffcf0c1ce0] +[a31n05:158577] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n05:158577] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n05:158577] [ 4] [a29n10:164941] *** Process received signal *** +[a29n10:164941] Signal: Segmentation fault (11) +[a29n10:164941] Signal code: (3) +[a29n10:164941] Failing at address: 0x14170000000000a0 +[a29n10:164941] [ 0] [0x1000000504d8] +[a29n10:164941] [ 1] [0x3fffd9db3510] +[a29n10:164941] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n10:164941] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n08:118555] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n08:118555] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n08:118555] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n08:118555] [10] [a26n18:69154] *** Process received signal *** +[a26n18:69154] Signal: Segmentation fault (11) +[a26n18:69154] Signal code: (3) +[a26n18:69154] Failing at address: 0x14170000000000a0 +[a26n18:69154] [ 0] [0x1000000504d8] +[a26n18:69154] [ 1] [0x3fffd8dbdd30] +[a26n18:69154] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n18:69154] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n18:69154] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n18:69154] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n10:140993] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n10:140993] [10] [b02n03:39688] *** Process received signal *** +[b02n03:39688] Signal: Segmentation fault (11) +[b02n03:39688] Signal code: (3) +[b02n03:39688] Failing at address: 0x14170000000000a0 +[b02n03:39688] [ 0] [0x1000000504d8] +[b02n03:39688] [ 1] [0x3fffdcffabb0] +[b02n03:39688] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n03:39688] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n03:39688] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n12:167679] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n12:167679] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n12:167679] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n12:167679] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n12:167679] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n07:163255] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n07:163255] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n13:137746] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n13:137746] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n07:124353] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n07:124353] *** End of error message *** +[b12n01:145735] *** Process received signal *** +[b12n01:145735] Signal: Segmentation fault (11) +[b12n01:145735] Signal code: (3) +[b12n01:145735] Failing at address: 0x14170000000000a0 +[b12n01:145735] [ 0] [0x1000000504d8] +[b12n01:145735] [ 1] [0x3fffe17026a0] +[b12n01:145735] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n01:145735] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n08:165283] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n08:165283] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n08:165283] *** End of error message *** +[b10n12:165846] *** Process received signal *** +[b10n12:165846] Signal: Segmentation fault (11) +[b10n12:165846] Signal code: (3) +[b10n12:165846] Failing at address: 0xb170000000000a0 +[b10n12:165846] [ 0] [0x1000000504d8] +[b10n12:165846] [ 1] [0x3fffcf437420] +[b10n12:165846] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n12:165846] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n12:165846] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n12:165846] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n12:132101] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n12:132101] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n12:132101] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n12:132101] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n07:08399] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n07:08399] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n07:08399] [13] [b07n05:176912] *** Process received signal *** +[b07n05:176912] Signal: Segmentation fault (11) +[b07n05:176912] Signal code: (3) +[b07n05:176912] Failing at address: 0x14170000000000a0 +[b07n05:176912] [ 0] [0x1000000504d8] +[b07n05:176912] [ 1] [0x3fffd7670410] +[b07n05:176912] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n05:176912] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n02:157955] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n02:157955] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n02:157955] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n02:157955] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n11:130808] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n11:130808] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n11:130808] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n11:130808] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n11:130808] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n11:130808] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n10:162612] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n10:162612] *** End of error message *** +[b10n05:173083] *** Process received signal *** +[b10n05:173083] Signal: Segmentation fault (11) +[b10n05:173083] Signal code: (3) +[b10n05:173083] Failing at address: 0x14170000000000a0 +[b10n05:173083] [ 0] [0x1000000504d8] +[b10n05:173083] [ 1] [0x3fffe4664630] +[b10n05:173083] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n05:173083] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n15:55852] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n15:55852] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n15:55852] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n15:55852] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n15:55852] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n15:55852] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n15:55852] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n18:145744] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n18:145744] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n18:145744] [11] [a27n11:23066] *** Process received signal *** +[a27n11:23066] Signal: Segmentation fault (11) +[a27n11:23066] Signal code: (3) +[a27n11:23066] Failing at address: 0x14170000000000a0 +[a27n11:23066] [ 0] [0x1000000504d8] +[a27n11:23066] [ 1] [0x3fffca036fe0] +[a27n11:23066] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n11:23066] [ 3] [b13n13:133625] *** Process received signal *** +[b13n13:133625] Signal: Segmentation fault (11) +[b13n13:133625] Signal code: (3) +[b13n13:133625] Failing at address: 0x14170000000000a0 +[b13n13:133625] [ 0] [0x1000000504d8] +[b13n13:133625] [ 1] [0x3fffdd94f3b0] +[b13n13:133625] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n13:133625] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n05:173824] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n05:173824] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n05:173824] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n04:168492] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n04:168492] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n04:168492] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n04:168492] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n15:177823] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n15:177823] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n15:177823] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n15:177823] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n15:177823] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n15:177823] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n15:177823] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n15:177823] [13] [b02n18:03313] *** Process received signal *** +[b02n18:03313] Signal: Segmentation fault (11) +[b02n18:03313] Signal code: (3) +[b02n18:03313] Failing at address: 0x14170000000000a0 +[b02n18:03313] [ 0] [0x1000000504d8] +[b02n18:03313] [ 1] [0x3fffd84cd8d0] +[b02n18:03313] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n18:03313] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n12:122965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n12:122965] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n12:122965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n12:122965] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n15:120134] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n15:120134] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n15:120134] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n15:120134] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n08:116173] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n08:116173] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n08:116173] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n08:116173] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n08:116173] [ 8] [b11n17:164295] *** Process received signal *** +[b11n17:164295] Signal: Segmentation fault (11) +[b11n17:164295] Signal code: (3) +[b11n17:164295] Failing at address: 0xb170000000000a0 +[b11n17:164295] [ 0] [0x1000000504d8] +[b11n17:164295] [ 1] [0x3fffe6047160] +[b11n17:164295] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n17:164295] [ 3] [b20n05:121823] *** Process received signal *** +[b20n05:121823] Signal: Segmentation fault (11) +[b20n05:121823] Signal code: (3) +[b20n05:121823] Failing at address: 0xb170000000000a0 +[b20n05:121823] [ 0] [0x1000000504d8] +[b20n05:121823] [ 1] [0x3fffc3a0d430] +[b20n05:121823] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n05:121823] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n05:121823] [ 4] [b06n04:175998] *** Process received signal *** +[b06n04:175998] Signal: Segmentation fault (11) +[b06n04:175998] Signal code: (3) +[b06n04:175998] Failing at address: 0xb170000000000a0 +[b06n04:175998] [ 0] [0x1000000504d8] +[b06n04:175998] [ 1] [0x3fffe75cb420] +[b06n04:175998] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n04:175998] [ 3] [b21n04:118346] *** Process received signal *** +[b21n04:118346] Signal: Segmentation fault (11) +[b21n04:118346] Signal code: (3) +[b21n04:118346] Failing at address: 0x14170000000000a0 +[b21n04:118346] [ 0] [0x1000000504d8] +[b21n04:118346] [ 1] [0x3ffffd922cf0] +[b21n04:118346] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n04:118346] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n04:118346] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n04:118346] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n18:121305] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n18:121305] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n18:69154] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n18:69154] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n18:69154] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n18:69154] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n18:69154] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n18:69154] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n18:69154] [12] [a34n07:165093] *** Process received signal *** +[a34n07:165093] Signal: Segmentation fault (11) +[a34n07:165093] Signal code: (3) +[a34n07:165093] Failing at address: 0x14170000000000a0 +[a34n07:165093] [ 0] [0x1000000504d8] +[a34n07:165093] [ 1] [0x3fffd367f280] +[a34n07:165093] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n07:165093] [ 3] [a31n08:156373] *** Process received signal *** +[a31n08:156373] Signal: Segmentation fault (11) +[a31n08:156373] Signal code: (3) +[a31n08:156373] Failing at address: 0x14170000000000a0 +[a31n08:156373] [ 0] [0x1000000504d8] +[a31n08:156373] [ 1] [0x3ffff26f32c0] +[a31n08:156373] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n08:156373] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n12:145264] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n12:145264] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n12:145264] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n12:145264] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n12:145264] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n12:145264] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n15:148581] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n15:148581] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n15:148581] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n15:148581] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n15:148581] [ 9] [a27n17:15875] *** Process received signal *** +[a27n17:15875] Signal: Segmentation fault (11) +[a27n17:15875] Signal code: (3) +[a27n17:15875] Failing at address: 0x14170000000000a0 +[a27n17:15875] [ 0] [0x1000000504d8] +[a27n17:15875] [ 1] [0x3ffffa6ef970] +[a27n17:15875] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n17:15875] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n17:129931] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n17:129931] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n17:129931] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n17:129931] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n08:159023] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n08:159023] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n08:159023] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n08:159023] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n08:159023] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n08:159023] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n08:159023] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n12:138739] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n12:138739] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n12:138739] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n12:138739] [13] [b05n04:06834] *** Process received signal *** +[b05n04:06834] Signal: Segmentation fault (11) +[b05n04:06834] Signal code: (3) +[b05n04:06834] Failing at address: 0x14170000000000a0 +[b05n04:06834] [ 0] [0x1000000504d8] +[b05n04:06834] [ 1] [0x3ffff0d734a0] +[b05n04:06834] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n04:06834] [ 3] [b08n06:161722] *** Process received signal *** +[b08n06:161722] Signal: Segmentation fault (11) +[b08n06:161722] Signal code: (3) +[b08n06:161722] Failing at address: 0xb170000000000a0 +[b08n06:161722] [ 0] [0x1000000504d8] +[b08n06:161722] [ 1] [0x3fffde86e570] +[b08n06:161722] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n06:161722] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n01:167299] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n01:167299] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n01:167299] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n01:167299] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n01:167299] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n16:168051] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n16:168051] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n16:168051] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n16:168051] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n11:128955] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n11:128955] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n11:128955] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n11:128955] [14] [b02n15:169495] *** Process received signal *** +[b02n15:169495] Signal: Segmentation fault (11) +[b02n15:169495] Signal code: (3) +[b02n15:169495] Failing at address: 0x14170000000000a0 +[b02n15:169495] [ 0] [0x1000000504d8] +[b02n15:169495] [ 1] [0x3ffff3103e60] +[b02n15:169495] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n15:169495] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n13:123056] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n13:123056] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n13:117512] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n13:117512] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n13:117512] [13] [b03n10:02713] *** Process received signal *** +[b03n10:02713] Signal: Segmentation fault (11) +[b03n10:02713] Signal code: (3) +[b03n10:02713] Failing at address: 0x14170000000000a0 +[b03n10:02713] [ 0] [0x1000000504d8] +[b03n10:02713] [ 1] [0x3ffffec803a0] +[b03n10:02713] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n10:02713] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n10:02713] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n02:157955] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n02:157955] [11] [b01n07:03461] *** Process received signal *** +[b01n07:03461] Signal: Segmentation fault (11) +[b01n07:03461] Signal code: (3) +[b01n07:03461] Failing at address: 0x14170000000000a0 +[a30n08:163098] *** Process received signal *** +[a30n08:163098] Signal: Segmentation fault (11) +[a30n08:163098] Signal code: (3) +[a30n08:163098] Failing at address: 0x14170000000000a0 +[a30n08:163098] [ 0] [0x1000000504d8] +[a30n08:163098] [ 1] [0x3fffdb4f2c10] +[a30n08:163098] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n08:163098] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n08:163098] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n11:152989] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n11:152989] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n11:152989] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n11:152989] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n11:152989] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n11:152989] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n11:152989] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n11:152989] [13] [a31n15:155434] *** Process received signal *** +[a31n15:155434] Signal: Segmentation fault (11) +[a31n15:155434] Signal code: (3) +[a31n15:155434] Failing at address: 0x14170000000000a0 +[a31n15:155434] [ 0] [0x1000000504d8] +[a31n15:155434] [ 1] [0x3fffef87a320] +[a31n15:155434] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n15:155434] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n15:55852] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n15:55852] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n15:55852] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n15:55852] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n15:55852] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n12:168646] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n12:168646] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n12:168646] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n12:168646] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n05:173824] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n05:173824] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n05:173824] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n05:173824] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n04:126862] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n04:126862] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n04:126862] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n04:126862] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n07:08399] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n07:08399] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n09:121756] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n09:121756] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n09:121756] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n10:140993] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n10:140993] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n10:140993] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n10:140993] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n12:122965] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n12:122965] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n12:167679] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n12:167679] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n12:167679] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n15:85941] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n15:85941] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n15:85941] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n15:85941] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n08:172901] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n08:172901] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n08:172901] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n08:172901] [10] [b21n15:119256] *** Process received signal *** +[b21n15:119256] Signal: Segmentation fault (11) +[b21n15:119256] Signal code: (3) +[b21n15:119256] Failing at address: 0x14170000000000a0 +[b21n15:119256] [ 0] [0x1000000504d8] +[b21n15:119256] [ 1] [0x3fffea912d50] +[b21n15:119256] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n15:119256] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n15:154387] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n15:154387] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n15:154387] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n15:154387] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n15:154387] [ 8] [b15n05:120334] *** Process received signal *** +[b15n05:120334] Signal: Segmentation fault (11) +[b15n05:120334] Signal code: (3) +[b15n05:120334] Failing at address: 0x14170000000000a0 +[b15n05:120334] [ 0] [0x1000000504d8] +[b15n05:120334] [ 1] [0x3fffdfbb75f0] +[b15n05:120334] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n05:120334] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n05:120334] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n05:120334] [ 5] [a32n07:172878] *** Process received signal *** +[a32n07:172878] Signal: Segmentation fault (11) +[a32n07:172878] Signal code: (3) +[a32n07:172878] Failing at address: 0xb170000000000a0 +[a32n07:172878] [ 0] [0x1000000504d8] +[a32n07:172878] [ 1] [0x3ffff34cdc60] +[a32n07:172878] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n07:172878] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n07:172878] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n12:160192] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n12:160192] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n12:160192] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n12:160192] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n12:160192] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n13:168163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n13:168163] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n13:168163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n13:168163] [ 8] [b14n02:139025] *** Process received signal *** +[b14n02:139025] Signal: Segmentation fault (11) +[b14n02:139025] Signal code: (3) +[b14n02:139025] Failing at address: 0xb170000000000a0 +[b14n02:139025] [ 0] [0x1000000504d8] +[b14n02:139025] [ 1] [0x3fffebfe81a0] +[b14n02:139025] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n02:139025] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n02:139025] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n08:118555] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n08:118555] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n08:118555] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n08:118555] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n13:154232] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n13:154232] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n13:154232] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n13:154232] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n04:168492] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n18:69154] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n18:69154] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n18:69154] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n18:162424] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n18:162424] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n18:162424] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n18:162424] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n18:162424] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n07:165093] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n07:165093] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n07:165093] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n18:149118] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n18:149118] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n18:149118] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n18:149118] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n11:130808] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n11:130808] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n11:130808] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n11:130808] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n04:118346] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n04:118346] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n04:118346] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n04:118346] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n04:118346] [10] [b17n15:137009] *** Process received signal *** +[b17n15:137009] Signal: Segmentation fault (11) +[b17n15:137009] Signal code: (3) +[b17n15:137009] Failing at address: 0xb170000000000a0 +[b17n15:137009] [ 0] [0x1000000504d8] +[b17n15:137009] [ 1] [0x3fffdfd1c0b0] +[b17n15:137009] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n15:137009] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n15:137009] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n01:145735] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n01:145735] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n01:145735] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n05:176912] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n05:176912] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n05:176912] [ 6] [a32n04:153967] *** Process received signal *** +[a32n04:153967] Signal: Segmentation fault (11) +[a32n04:153967] Signal code: (3) +[a32n04:153967] Failing at address: 0x5170000000000a0 +[a32n04:153967] [ 0] [0x1000000504d8] +[a32n04:153967] [ 1] [0x3ffffc490ca0] +[a32n04:153967] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n04:153967] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n18:145744] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n18:145744] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n18:145744] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n18:145744] *** End of error message *** +[a27n05:53377] *** Process received signal *** +[a27n05:53377] Signal: Segmentation fault (11) +[a27n05:53377] Signal code: (3) +[a27n05:53377] Failing at address: 0x14170000000000a0 +[a27n05:53377] [ 0] [0x1000000504d8] +[a27n05:53377] [ 1] [0x3ffffe909980] +[a27n05:53377] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n05:53377] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n05:53377] [ 4] [b10n03:168931] *** Process received signal *** +[b10n03:168931] Signal: Segmentation fault (11) +[b10n03:168931] Signal code: (3) +[b10n03:168931] Failing at address: 0xb170000000000a0 +[b10n03:168931] [ 0] [0x1000000504d8] +[b10n03:168931] [ 1] [0x3fffe8e407c0] +[b10n03:168931] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n03:168931] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n03:168931] [ 4] [a29n03:155730] *** Process received signal *** +[a29n03:155730] Signal: Segmentation fault (11) +[a29n03:155730] Signal code: (3) +[a29n03:155730] Failing at address: 0xb170000000000a0 +[a29n03:155730] [ 0] [0x1000000504d8] +[a29n03:155730] [ 1] [0x3fffd25f27e0] +[a29n03:155730] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n03:155730] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n17:15875] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n17:15875] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n17:15875] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n17:15875] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n12:132101] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n12:132101] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n12:132101] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n12:132101] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n04:126346] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n04:126346] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n04:126346] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n04:126346] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n04:126346] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n04:126346] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n04:126346] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n04:126346] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n04:129408] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n04:129408] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n04:129408] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n04:129408] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n04:129408] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n04:129408] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n04:129408] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n04:129408] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n04:129408] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n04:129408] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n04:129408] *** End of error message *** +[a31n13:155849] *** Process received signal *** +[a31n13:155849] Signal: Segmentation fault (11) +[a31n13:155849] Signal code: (3) +[a31n13:155849] Failing at address: 0xb170000000000a0 +[a31n13:155849] [ 0] [0x1000000504d8] +[a31n13:155849] [ 1] [0x3fffd35abad0] +[a31n13:155849] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n13:155849] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n13:155849] [ 4] [a33n07:149480] *** Process received signal *** +[a33n07:149480] Signal: Segmentation fault (11) +[a33n07:149480] Signal code: (3) +[a33n07:149480] Failing at address: 0x14170000000000a0 +[a33n07:149480] [ 0] [0x1000000504d8] +[a33n07:149480] [ 1] [0x3ffffb9caf00] +[a33n07:149480] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n07:149480] [ 3] [b22n02:119532] *** Process received signal *** +[b22n02:119532] Signal: Segmentation fault (11) +[b22n02:119532] Signal code: (3) +[b22n02:119532] Failing at address: 0xb170000000000a0 +[b22n02:119532] [ 0] [0x1000000504d8] +[b22n02:119532] [ 1] [0x3ffff22e9dd0] +[b22n02:119532] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n02:119532] [ 3] [b23n10:126790] *** Process received signal *** +[b23n10:126790] Signal: Segmentation fault (11) +[b23n10:126790] Signal code: (3) +[b23n10:126790] Failing at address: 0xb170000000000a0 +[b23n10:126790] [ 0] [0x1000000504d8] +[b23n10:126790] [ 1] [0x3fffcb2a2ee0] +[b23n10:126790] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n10:126790] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n10:126790] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n02:157955] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n02:157955] [13] [b01n07:03461] [ 0] [0x1000000504d8] +[b01n07:03461] [ 1] [0x3fffc0ad5a30] +[b01n07:03461] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n07:03461] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n07:03461] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n07:03461] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n07:03461] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n16:168051] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n16:168051] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n13:117512] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n13:117512] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n12:145264] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n12:145264] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n12:145264] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n08:159023] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n08:159023] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n08:159023] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n08:159023] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n08:159023] *** End of error message *** +[b09n05:173824] *** End of error message *** +[b09n08:176681] *** Process received signal *** +[b09n08:176681] Signal: Segmentation fault (11) +[b09n08:176681] Signal code: (3) +[b09n08:176681] Failing at address: 0xb170000000000a0 +[b09n08:176681] [ 0] [0x1000000504d8] +[b09n08:176681] [ 1] [0x3fffe85f1150] +[b09n08:176681] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n08:176681] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n08:176681] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n04:06834] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n04:06834] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n04:06834] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n04:06834] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n04:06834] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n04:06834] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n04:06834] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n04:06834] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n04:06834] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n04:06834] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n06:161722] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n06:161722] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n06:161722] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n06:161722] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n06:161722] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n06:161722] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n06:161722] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n06:161722] [11] [b18n05:135121] *** Process received signal *** +[b18n05:135121] Signal: Segmentation fault (11) +[b18n05:135121] Signal code: (3) +[b18n05:135121] Failing at address: 0xb170000000000a0 +[b18n05:135121] [ 0] [0x1000000504d8] +[b18n05:135121] [ 1] [0x3fffe7004a80] +[b18n05:135121] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n05:135121] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n12:167679] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n12:167679] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n12:167679] *** End of error message *** +[b08n08:164753] *** Process received signal *** +[b08n08:164753] Signal: Segmentation fault (11) +[b08n08:164753] Signal code: (3) +[b08n08:164753] Failing at address: 0x14170000000000a0 +[b08n08:164753] [ 0] [0x1000000504d8] +[b08n08:164753] [ 1] [0x3fffe7c059c0] +[b08n08:164753] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n08:164753] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n08:164753] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n04:175998] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n04:175998] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n04:175998] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n04:175998] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n15:120134] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n15:120134] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n08:172901] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n08:172901] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n08:172901] [13] [b23n09:119636] *** Process received signal *** +[b23n09:119636] Signal: Segmentation fault (11) +[b23n09:119636] Signal code: (3) +[b23n09:119636] Failing at address: 0x14170000000000a0 +[b23n09:119636] [ 0] [0x1000000504d8] +[b23n09:119636] [ 1] [0x3fffe16d7fa0] +[b23n09:119636] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n09:119636] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n09:119636] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n16:116527] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n16:116527] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n16:116527] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n02:155381] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n02:155381] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n02:155381] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n02:155381] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n02:155381] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n02:155381] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n02:155381] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n02:155381] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n17:164295] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n17:164295] [ 5] [b19n16:130378] *** Process received signal *** +[b19n16:130378] Signal: Segmentation fault (11) +[b19n16:130378] Signal code: (3) +[b19n16:130378] Failing at address: 0x14170000000000a0 +[b19n16:130378] [ 0] [0x1000000504d8] +[b19n16:130378] [ 1] [0x3fffd5ee09e0] +[b19n16:130378] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n16:130378] [ 3] [b04n15:177859] *** Process received signal *** +[b04n15:177859] Signal: Segmentation fault (11) +[b04n15:177859] Signal code: (3) +[b04n15:177859] Failing at address: 0xb170000000000a0 +[b04n15:177859] [ 0] [0x1000000504d8] +[b04n15:177859] [ 1] [0x3fffda58bac0] +[b04n15:177859] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n15:177859] [ 3] [a31n06:167369] *** Process received signal *** +[a31n06:167369] Signal: Segmentation fault (11) +[a31n06:167369] Signal code: (3) +[a31n06:167369] Failing at address: 0xb170000000000a0 +[a31n06:167369] [ 0] [0x1000000504d8] +[a31n06:167369] [ 1] [0x3fffc2843c10] +[a31n06:167369] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n06:167369] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n01:167299] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n01:167299] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n01:167299] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n01:167299] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n10:164941] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n10:164941] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n10:164941] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n10:164941] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n10:164941] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n10:164941] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n10:164941] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n10:164941] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n10:164941] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n10:164941] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n11:23066] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n11:23066] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n11:23066] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n11:23066] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n11:23066] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n11:23066] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n11:23066] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n11:23066] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n11:23066] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n11:23066] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n01:128006] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n01:128006] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n01:128006] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n01:128006] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n12:138739] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n12:138739] *** End of error message *** +[a26n18:69157] *** Process received signal *** +[a26n18:69157] Signal: Segmentation fault (11) +[a26n18:69157] Signal code: (3) +[a26n18:69157] Failing at address: 0x14170000000000a0 +[a26n18:69157] [ 0] [0x1000000504d8] +[a26n18:69157] [ 1] [b20n13:131760] *** Process received signal *** +[b20n13:131760] Signal: Segmentation fault (11) +[b20n13:131760] Signal code: (3) +[b20n13:131760] Failing at address: 0xb170000000000a0 +[b20n13:131760] [ 0] [0x1000000504d8] +[b20n13:131760] [ 1] [0x3ffff1a3b000] +[b20n13:131760] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n13:131760] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n07:172878] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n07:172878] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n07:172878] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n07:172878] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n07:165093] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n07:165093] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n07:165093] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n07:165093] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n18:121305] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n18:121305] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n18:121305] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n18:121305] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n15:169495] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n15:169495] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n15:169495] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n15:169495] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n15:169495] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n15:169495] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n15:169495] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n15:169495] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n15:169495] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n15:169495] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n07:163255] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n07:163255] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n07:163255] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n07:163255] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n07:163255] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n11:130808] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n11:130808] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n04:118346] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n04:118346] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n04:118346] [13] [a32n05:154187] *** Process received signal *** +[a32n05:154187] Signal: Segmentation fault (11) +[a32n05:154187] Signal code: (3) +[a32n05:154187] Failing at address: 0xb170000000000a0 +[a32n05:154187] [ 0] [0x1000000504d8] +[a32n05:154187] [ 1] [0x3fffdddae980] +[a32n05:154187] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n05:154187] [ 3] [b06n06:163676] *** Process received signal *** +[b06n06:163676] Signal: Segmentation fault (11) +[b06n06:163676] Signal code: (3) +[b06n06:163676] Failing at address: 0xb170000000000a0 +[b06n06:163676] [ 0] [0x1000000504d8] +[b06n06:163676] [ 1] [0x3fffd1ecab50] +[b06n06:163676] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n06:163676] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n12:165846] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n12:165846] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n12:165846] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n12:165846] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n12:165846] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n12:165846] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n12:165846] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n12:165846] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n10:02713] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n10:02713] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n10:02713] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n10:02713] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n10:02713] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n10:02713] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n10:02713] [11] [b16n14:133634] *** Process received signal *** +[b16n14:133634] Signal: Segmentation fault (11) +[b16n14:133634] Signal code: (3) +[b16n14:133634] Failing at address: 0xb170000000000a0 +[b16n14:133634] [ 0] [0x1000000504d8] +[b16n14:133634] [ 1] [0x3fffec62eb10] +[b16n14:133634] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n14:133634] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n14:133634] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n03:158657] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n03:158657] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n03:158657] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n03:158657] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n15:85941] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n15:85941] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n15:85941] [11] [b09n12:178719] *** Process received signal *** +[b09n12:178719] Signal: Segmentation fault (11) +[b09n12:178719] Signal code: (3) +[b09n12:178719] Failing at address: 0x14170000000000a0 +[b09n12:178719] [ 0] [0x1000000504d8] +[b09n12:178719] [ 1] [0x3fffddff5820] +[b09n12:178719] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n12:178719] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n12:178719] [ 4] [b04n18:176813] *** Process received signal *** +[b04n18:176813] Signal: Segmentation fault (11) +[b04n18:176813] Signal code: (3) +[b04n18:176813] Failing at address: 0x5170000000000a0 +[b04n18:176813] [ 0] [0x1000000504d8] +[b04n18:176813] [ 1] [0x3fffecd54d40] +[b04n18:176813] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n18:176813] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n05:53377] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n05:53377] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n05:53377] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n05:53377] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n17:15875] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n17:15875] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n17:15875] [10] [b18n02:125375] *** Process received signal *** +[b18n02:125375] Signal: Segmentation fault (11) +[b18n02:125375] Signal code: (3) +[b18n02:125375] Failing at address: 0x14170000000000a0 +[b18n02:125375] [ 0] [0x1000000504d8] +[b18n02:125375] [ 1] [0x3ffffa7b7bd0] +[b18n02:125375] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n02:125375] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n13:123056] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n13:123056] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n13:123056] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n13:123056] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n13:123056] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n10:126790] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n10:126790] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n10:126790] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n12:145264] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n12:145264] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n18:03313] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n18:03313] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n18:03313] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n18:03313] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n18:03313] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n18:03313] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n18:03313] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n15:119256] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n15:119256] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n15:119256] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n15:119256] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n15:119256] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n07:03461] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n07:03461] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n07:03461] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n07:03461] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n02:157955] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n02:157955] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n02:119532] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n02:119532] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n02:119532] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n10:140993] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n05:173083] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n05:173083] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n05:173083] [ 6] [b16n09:124194] *** Process received signal *** +[b16n09:124194] Signal: Segmentation fault (11) +[b16n09:124194] Signal code: (3) +[b16n09:124194] Failing at address: 0x14170000000000a0 +[b16n09:124194] [ 0] [0x1000000504d8] +[b16n09:124194] [ 1] [0x3fffcb6ad310] +[b16n09:124194] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n09:124194] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n09:124194] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n01:128832] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n01:128832] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n01:128832] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n01:128832] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n01:128832] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n01:128832] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n05:120334] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n05:120334] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n05:120334] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n05:120334] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n05:120334] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n05:120334] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n05:120334] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n05:120334] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n05:120334] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n05:120334] *** End of error message *** +[a29n05:174783] *** Process received signal *** +[a29n05:174783] Signal: Segmentation fault (11) +[a29n05:174783] Signal code: (3) +[a29n05:174783] Failing at address: 0x14170000000000a0 +[a29n05:174783] [ 0] [0x1000000504d8] +[a29n05:174783] [ 1] [0x3fffe2a52a20] +[a29n05:174783] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n05:174783] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n05:174783] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n09:121756] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n09:121756] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n09:121756] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n09:121756] *** End of error message *** +[b17n18:126905] *** Process received signal *** +[b17n18:126905] Signal: Segmentation fault (11) +[b17n18:126905] Signal code: (3) +[b17n18:126905] Failing at address: 0x14170000000000a0 +[b17n18:126905] [ 0] [0x1000000504d8] +[b17n18:126905] [ 1] [0x3fffe07bdbd0] +[b17n18:126905] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n18:126905] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n01:167299] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n01:167299] *** End of error message *** +[b01n01:146657] *** Process received signal *** +[b01n01:146657] Signal: Segmentation fault (11) +[b01n01:146657] Signal code: (3) +[b01n01:146657] Failing at address: 0xb170000000000a0 +[b01n01:146657] [ 0] [0x1000000504d8] +[b01n01:146657] [ 1] [0x3fffe2ae4c70] +[b01n01:146657] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n01:146657] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n01:145735] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n01:145735] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n01:145735] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n01:145735] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n17:169739] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n17:169739] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n17:169739] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n17:169739] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n17:169739] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n17:169739] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n17:169739] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n17:169739] [12] [a27n15:32403] *** Process received signal *** +[a27n15:32403] Signal: Segmentation fault (11) +[a27n15:32403] Signal code: (3) +[a27n15:32403] Failing at address: 0x14170000000000a0 +[a27n15:32403] [ 0] [0x1000000504d8] +[a27n15:32403] [ 1] [0x3ffff84aed90] +[a27n15:32403] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n15:32403] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n15:32403] [ 4] [b13n12:140122] *** Process received signal *** +[b13n12:140122] Signal: Segmentation fault (11) +[b13n12:140122] Signal code: (3) +[b13n12:140122] Failing at address: 0xb170000000000a0 +[b13n12:140122] [ 0] [0x1000000504d8] +[b13n12:140122] [ 1] [0x3ffff57aa060] +[b13n12:140122] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n12:140122] [ 3] [0x3ffff48245f0] +[a26n18:69157] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n18:69157] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n18:69157] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n07:165093] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n07:165093] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n07:165093] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n07:165093] [14] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n08:118555] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n16:116527] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n16:116527] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n16:116527] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n16:116527] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n15:177859] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n15:177859] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n15:177859] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n15:177859] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n15:177859] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n15:177859] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n15:177859] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n15:177859] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n15:177859] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n08:164753] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n08:164753] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n08:164753] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n08:164753] [ 8] [b04n02:177550] *** Process received signal *** +[b04n02:177550] Signal: Segmentation fault (11) +[b04n02:177550] Signal code: (3) +[b04n02:177550] Failing at address: 0xb170000000000a0 +[b04n02:177550] [ 0] [0x1000000504d8] +[b04n02:177550] [ 1] [0x3ffffc47c810] +[b04n02:177550] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n02:177550] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n02:177550] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n08:116173] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n08:116173] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n08:116173] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n08:116173] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n08:116173] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n13:131760] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n13:131760] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n13:131760] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n13:131760] [ 7] [a36n03:153851] *** Process received signal *** +[a36n03:153851] Signal: Segmentation fault (11) +[a36n03:153851] Signal code: (3) +[a36n03:153851] Failing at address: 0xb170000000000a0 +[a36n03:153851] [ 0] [0x1000000504d8] +[a36n03:153851] [ 1] [0x3fffe7e0eb80] +[a36n03:153851] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n03:153851] [ 3] [a36n10:142931] *** Process received signal *** +[a36n10:142931] Signal: Segmentation fault (11) +[a36n10:142931] Signal code: (3) +[a36n10:142931] Failing at address: 0xb170000000000a0 +[a36n10:142931] [ 0] [0x1000000504d8] +[a36n10:142931] [ 1] [0x3fffc9375e20] +[a36n10:142931] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n10:142931] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n10:142931] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n11:152989] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n11:152989] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n05:158577] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n13:133625] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n13:133625] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n13:133625] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n13:133625] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n13:133625] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n04:06834] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n04:06834] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n18:176813] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n18:176813] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n18:176813] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n05:53377] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n05:53377] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n15:177823] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n15:177823] *** End of error message *** +[b18n10:122207] *** Process received signal *** +[b18n10:122207] Signal: Segmentation fault (11) +[b18n10:122207] Signal code: (3) +[b18n10:122207] Failing at address: 0xb170000000000a0 +[b18n10:122207] [ 0] [0x1000000504d8] +[b18n10:122207] [ 1] [0x3ffff3fd6c00] +[b18n10:122207] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n10:122207] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n04:175998] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n04:175998] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n04:175998] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n04:175998] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n04:175998] [12] [b01n12:178598] *** Process received signal *** +[b01n12:178598] Signal: Segmentation fault (11) +[b01n12:178598] Signal code: (3) +[b01n12:178598] Failing at address: 0xb170000000000a0 +[b01n12:178598] [ 0] [0x1000000504d8] +[b01n12:178598] [ 1] [0x3fffd5fa1640] +[b01n12:178598] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n12:178598] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n12:178598] [ 4] [b03n15:85944] *** Process received signal *** +[b03n15:85944] Signal: Segmentation fault (11) +[b03n15:85944] Signal code: (3) +[b03n15:85944] Failing at address: 0x5170000000000a0 +[b03n15:85944] [ 0] [0x1000000504d8] +[b03n15:85944] [ 1] [0x3fffe20691a0] +[b03n15:85944] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n15:85944] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n05:176912] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n05:176912] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n05:176912] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n05:176912] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n08:163098] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n08:163098] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n08:163098] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n08:163098] [ 8] [b09n18:164798] *** Process received signal *** +[b09n18:164798] Signal: Segmentation fault (11) +[b09n18:164798] Signal code: (3) +[b09n18:164798] Failing at address: 0xb170000000000a0 +[b09n18:164798] [ 0] [0x1000000504d8] +[b09n18:164798] [ 1] [0x3fffe24dabd0] +[b09n18:164798] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n18:164798] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n18:164798] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n07:127356] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n07:127356] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n07:127356] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n07:127356] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n07:127356] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n07:127356] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n07:127356] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n04:118346] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n04:118346] *** End of error message *** +[a32n02:157953] *** Process received signal *** +[a32n02:157953] Signal: Segmentation fault (11) +[a32n02:157953] Signal code: (3) +[a32n02:157953] Failing at address: 0x14170000000000a0 +[a32n02:157953] [ 0] [0x1000000504d8] +[a32n02:157953] [ 1] [0x3ffff5a52860] +[a32n02:157953] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n02:119532] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n02:119532] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n02:119532] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n03:124683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n03:124683] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n03:124683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n03:124683] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n14:165666] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n14:165666] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n17:164295] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n17:164295] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n17:164295] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n17:164295] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n17:164295] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n15:148581] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n15:148581] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n15:148581] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n15:148581] [13] [a34n08:163162] *** Process received signal *** +[a34n08:163162] Signal: Segmentation fault (11) +[a34n08:163162] Signal code: (3) +[a34n08:163162] Failing at address: 0xb170000000000a0 +[a34n08:163162] [ 0] [0x1000000504d8] +[a34n08:163162] [ 1] [0x3ffffae30390] +[a34n08:163162] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n08:163162] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n03:155730] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n03:155730] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n03:155730] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n08:176681] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n08:176681] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n08:176681] [ 7] [b16n02:121684] *** Process received signal *** +[b16n02:121684] Signal: Segmentation fault (11) +[b16n02:121684] Signal code: (3) +[b16n02:121684] Failing at address: 0x14170000000000a0 +[b16n02:121684] [ 0] [0x1000000504d8] +[b16n02:121684] [ 1] [0x3fffead5ad50] +[b16n02:121684] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n02:121684] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n02:121684] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n12:178719] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n12:178719] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n12:178719] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n12:178719] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n17:15875] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n17:15875] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n17:15875] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n17:15875] [14] [b12n05:138198] *** Process received signal *** +[b12n05:138198] Signal: Segmentation fault (11) +[b12n05:138198] Signal code: (3) +[b12n05:138198] Failing at address: 0xb170000000000a0 +[b12n05:138198] [ 0] [0x1000000504d8] +[b12n05:138198] [ 1] [0x3fffcba5bb90] +[b12n05:138198] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n05:138198] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n05:138198] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n05:138198] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n05:138198] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n06:161722] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n06:161722] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n06:161722] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n06:161722] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n06:163676] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n06:163676] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n06:163676] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n06:163676] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n06:163676] [ 8] [b02n06:43020] *** Process received signal *** +[b02n06:43020] Signal: Segmentation fault (11) +[b02n06:43020] Signal code: (3) +[b02n06:43020] Failing at address: 0xb170000000000a0 +[b02n06:43020] [ 0] [0x1000000504d8] +[b02n06:43020] [ 1] [0x3fffe2fceca0] +[b02n06:43020] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n06:43020] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n18:121305] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n18:03313] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n18:03313] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n18:03313] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n18:03313] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n18:03313] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n14:133634] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n14:133634] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n14:133634] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n14:133634] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n08:172901] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n08:172901] *** End of error message *** +[b23n03:125244] *** Process received signal *** +[b23n03:125244] Signal: Segmentation fault (11) +[b23n03:125244] Signal code: (3) +[b23n03:125244] Failing at address: 0x14170000000000a0 +[b23n03:125244] [ 0] [0x1000000504d8] +[b23n03:125244] [ 1] [0x3fffdb0bf400] +[b23n03:125244] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n03:125244] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n07:172878] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n07:172878] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n07:172878] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n15:154387] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n15:154387] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n15:154387] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n15:154387] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n15:154387] [13] [a34n10:141235] *** Process received signal *** +[a34n10:141235] Signal: Segmentation fault (11) +[a34n10:141235] Signal code: (3) +[a34n10:141235] Failing at address: 0x14170000000000a0 +[a34n10:141235] [ 0] [0x1000000504d8] +[a34n10:141235] [ 1] [0x3fffe12d6cf0] +[a34n10:141235] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n10:141235] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n10:141235] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n07:03461] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n07:03461] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n07:03461] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n07:03461] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n07:03461] *** End of error message *** +[a36n02:147883] *** Process received signal *** +[a36n02:147883] Signal: Segmentation fault (11) +[a36n02:147883] Signal code: (3) +[a36n02:147883] Failing at address: 0xb170000000000a0 +[a36n02:147883] [ 0] [0x1000000504d8] +[a36n02:147883] [ 1] [a26n09:62482] *** Process received signal *** +[a26n09:62482] Signal: Segmentation fault (11) +[a26n09:62482] Signal code: (3) +[a26n09:62482] Failing at address: 0x14170000000000a0 +[a26n09:62482] [ 0] [0x1000000504d8] +[a26n09:62482] [ 1] [0x3fffefdebcf0] +[a26n09:62482] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n09:62482] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n12:168646] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n12:168646] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n12:168646] [11] [b08n02:162036] *** Process received signal *** +[b08n02:162036] Signal: Segmentation fault (11) +[b08n02:162036] Signal code: (3) +[b08n02:162036] Failing at address: 0x14170000000000a0 +[b08n02:162036] [ 0] [0x1000000504d8] +[b08n02:162036] [ 1] [0x3fffe41af100] +[b08n02:162036] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n02:162036] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n02:162036] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n18:69157] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n18:69157] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n07:165093] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n13:154232] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n13:154232] [10] [b02n09:166575] *** Process received signal *** +[b02n09:166575] Signal: Segmentation fault (11) +[b02n09:166575] Signal code: (3) +[b02n09:166575] Failing at address: 0x14170000000000a0 +[b02n09:166575] [ 0] [0x1000000504d8] +[b02n09:166575] [ 1] [0x3fffdb51cc10] +[b02n09:166575] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n09:166575] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n09:166575] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n15:169495] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n15:169495] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n09:124194] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n09:124194] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n09:124194] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n09:124194] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n09:124194] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n04:153967] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n04:153967] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n04:153967] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n04:153967] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n04:153967] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n04:153967] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n04:153967] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n04:153967] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n04:153967] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n15:119256] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n15:119256] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n15:119256] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n15:119256] [12] [b11n06:164018] *** Process received signal *** +[b11n06:164018] Signal: Segmentation fault (11) +[b11n06:164018] Signal code: (3) +[b11n06:164018] Failing at address: 0x14170000000000a0 +[b11n06:164018] [ 0] [0x1000000504d8] +[b11n06:164018] [ 1] [0x3fffeeead220] +[b11n06:164018] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n06:164018] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n13:168163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n13:168163] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n13:168163] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n04:126862] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n04:126862] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n04:126862] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n11:128955] *** End of error message *** +[a27n06:60151] *** Process received signal *** +[a27n06:60151] Signal: Segmentation fault (11) +[a27n06:60151] Signal code: (3) +[a27n06:60151] Failing at address: 0xb170000000000a0 +[a27n06:60151] [ 0] [0x1000000504d8] +[a27n06:60151] [ 1] [0x3ffffa399f70] +[a27n06:60151] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n06:60151] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n11:23066] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n11:23066] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n10:164941] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n10:164941] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n18:176813] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n18:176813] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n18:176813] [ 9] [b16n07:119637] *** Process received signal *** +[b16n07:119637] Signal: Segmentation fault (11) +[b16n07:119637] Signal code: (3) +[b16n07:119637] Failing at address: 0xb170000000000a0 +[b16n07:119637] [ 0] [0x1000000504d8] +[b16n07:119637] [ 1] [0x3fffc63d0e20] +[b16n07:119637] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n07:119637] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n07:119637] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n15:177859] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n15:177859] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n15:177859] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n05:53377] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n05:53377] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n05:53377] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n05:53377] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n05:53377] *** End of error message *** +[b19n05:131264] *** Process received signal *** +[b19n05:131264] Signal: Segmentation fault (11) +[b19n05:131264] Signal code: (3) +[b19n05:131264] Failing at address: 0x14170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n15:85941] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n15:85941] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n15:85941] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n15:85941] *** End of error message *** +[b23n06:117527] *** Process received signal *** +[b23n06:117527] Signal: Segmentation fault (11) +[b23n06:117527] Signal code: (3) +[b23n06:117527] Failing at address: 0xb170000000000a0 +[b23n06:117527] [ 0] [0x1000000504d8] +[b23n06:117527] [ 1] [0x3fffcf3834b0] +[b23n06:117527] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n06:117527] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n06:117527] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n16:116527] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n16:116527] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n16:116527] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n16:116527] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n13:131760] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n13:131760] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n13:131760] [10] [a36n06:143013] *** Process received signal *** +[a36n06:143013] Signal: Segmentation fault (11) +[a36n06:143013] Signal code: (3) +[a36n06:143013] Failing at address: 0x5170000000000a0 +[a36n06:143013] [ 0] [0x1000000504d8] +[a36n06:143013] [ 1] [0x3fffd80e8b90] +[a36n06:143013] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n06:143013] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n06:143013] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n18:162424] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n18:162424] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n18:162424] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n18:162424] [13] [b14n07:127359] *** Process received signal *** +[b14n07:127359] Signal: Segmentation fault (11) +[b14n07:127359] Signal code: (3) +[b14n07:127359] Failing at address: 0xb170000000000a0 +[b14n07:127359] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n05:121823] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n05:121823] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n05:121823] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n05:121823] [ 8] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n02:157953] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n02:157953] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n02:157953] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n02:157953] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n02:157953] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n02:157953] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n02:157953] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n02:157953] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n02:157953] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n02:157953] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n02:157953] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n02:157953] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n02:157953] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n03:153851] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n03:153851] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n03:153851] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n03:153851] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n03:153851] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n03:153851] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n03:153851] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n03:153851] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n02:119532] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n02:119532] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n02:119532] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n02:119532] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n18:149118] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n18:149118] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n18:149118] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n18:149118] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n18:149118] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n18:149118] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n18:149118] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n05:138198] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n05:138198] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n05:138198] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n05:138198] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n05:138198] [11] [b22n09:122070] *** Process received signal *** +[b22n09:122070] Signal: Segmentation fault (11) +[b22n09:122070] Signal code: (3) +[b22n09:122070] Failing at address: 0xb170000000000a0 +[b22n09:122070] [ 0] [0x1000000504d8] +[b22n09:122070] [ 1] [0x3fffd6ce6a10] +[b22n09:122070] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n09:122070] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n12:178719] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n12:178719] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n12:178719] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n04:175998] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n04:175998] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n04:175998] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n10:02713] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n10:02713] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n10:02713] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n10:02713] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n02:139025] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n02:139025] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n02:139025] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n02:139025] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n02:139025] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n02:139025] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n02:139025] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n02:139025] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n02:139025] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n05:176912] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n05:176912] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n05:176912] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n07:172878] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n07:172878] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n07:172878] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n01:145735] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n01:145735] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n01:145735] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n02:177550] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n02:177550] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n02:177550] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n02:177550] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n02:177550] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n02:177550] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n09:119636] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n09:119636] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n09:119636] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n09:119636] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n09:119636] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n05:154187] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n05:154187] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n05:154187] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n05:154187] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n05:154187] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n05:154187] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n05:154187] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n05:154187] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n05:154187] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n05:154187] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n03:168931] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n03:168931] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n03:168931] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n03:168931] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n08:163162] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n08:163162] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n08:163162] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n08:156373] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n08:156373] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n08:156373] [ 6] [a36n02:147886] *** Process received signal *** +[a36n02:147886] Signal: Segmentation fault (11) +[a36n02:147886] Signal code: (3) +[a36n02:147886] Failing at address: 0x5170000000000a0 +[a36n02:147886] [ 0] [0x1000000504d8] +[a36n02:147886] [ 1] [0x3ffff68fff50] +[a36n02:147886] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n02:147886] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n07:149480] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n07:149480] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n07:149480] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n07:149480] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n07:149480] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n07:149480] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n07:149480] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n07:149480] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n05:174783] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n05:174783] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n05:174783] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n05:174783] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n05:174783] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n05:174783] [10] [b04n16:08387] *** Process received signal *** +[b04n16:08387] Signal: Segmentation fault (11) +[b04n16:08387] Signal code: (3) +[b04n16:08387] Failing at address: 0x14170000000000a0 +[b04n16:08387] [ 0] [0x1000000504d8] +[b04n16:08387] [ 1] [0x3ffff019c4f0] +[b04n16:08387] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n16:08387] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n16:08387] [ 4] [b12n07:140137] *** Process received signal *** +[b12n07:140137] Signal: Segmentation fault (11) +[b12n07:140137] Signal code: (3) +[b12n07:140137] Failing at address: 0x14170000000000a0 +[b12n07:140137] [ 0] [0x1000000504d8] +[b12n07:140137] [ 1] [0x3fffcab38230] +[b12n07:140137] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n07:140137] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n07:140137] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n15:155434] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n15:155434] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n15:155434] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n15:155434] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n15:155434] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n15:155434] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n15:155434] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n15:155434] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n15:155434] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n15:155434] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n18:69157] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n18:69157] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n18:69157] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n18:69157] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n18:69157] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n18:69157] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n18:69157] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n18:69157] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n06:163676] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n06:163676] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n06:163676] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n06:163676] [12] [b19n03:123616] *** Process received signal *** +[b19n03:123616] Signal: Segmentation fault (11) +[b19n03:123616] Signal code: (3) +[b19n03:123616] Failing at address: 0xb170000000000a0 +[b19n03:123616] [ 0] [0x1000000504d8] +[b19n03:123616] [ 1] [0x3ffffedec480] +[b19n03:123616] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n03:123616] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n03:123616] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n14:133634] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n14:133634] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n15:32403] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n15:32403] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n15:32403] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n15:32403] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n02:121684] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n02:121684] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n02:121684] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n02:121684] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n17:164295] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n17:164295] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n17:164295] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n17:164295] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n15:137009] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n15:137009] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n15:137009] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n15:137009] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n15:137009] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n06:164018] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n06:164018] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n06:164018] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n06:164018] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n06:164018] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n10:126790] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n10:126790] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n10:126790] [10] [b01n06:171505] *** Process received signal *** +[b01n06:171505] Signal: Segmentation fault (11) +[b01n06:171505] Signal code: (3) +[b01n06:171505] Failing at address: 0x5170000000000a0 +[b01n06:171505] [ 0] [0x1000000504d8] +[b01n06:171505] [ 1] [0x3fffcddcbe80] +[b01n06:171505] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n06:171505] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n12:165846] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n12:165846] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n03:155730] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n03:155730] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n03:155730] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n03:155730] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n12:160192] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n12:160192] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n12:160192] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n12:160192] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n08:176681] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n08:176681] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n08:176681] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n08:164753] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n08:164753] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n08:164753] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n08:164753] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n18:176813] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n18:176813] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n18:176813] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n18:176813] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n12:178598] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n12:178598] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n12:178598] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n12:178598] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n09:62482] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n09:62482] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n09:62482] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n09:62482] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n09:124194] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n09:124194] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n09:124194] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n09:124194] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n09:124194] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n15:85944] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n15:85944] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n15:85944] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n15:85944] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n15:85944] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n15:85944] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n13:131760] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n13:131760] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n13:131760] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n13:131760] [14] [b14n16:122496] *** Process received signal *** +[b14n16:122496] Signal: Segmentation fault (11) +[b14n16:122496] Signal code: (3) +[b14n16:122496] Failing at address: 0x14170000000000a0 +[b14n16:122496] [ 0] [0x1000000504d8] +[b14n16:122496] [ 1] [0x3ffff14e8b00] +[b14n16:122496] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n16:122496] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n16:122496] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n15:119256] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n15:119256] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n15:119256] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n10:141235] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n10:141235] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n10:141235] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n10:141235] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n10:141235] [ 9] [0x1000000504d8] +[b14n07:127359] [ 1] [0x3fffe62ab2f0] +[b14n07:127359] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n07:127359] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n07:127359] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n07:127359] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n07:127359] [ 6] [a32n02:157954] *** Process received signal *** +[a32n02:157954] Signal: Segmentation fault (11) +[a32n02:157954] Signal code: (3) +[a32n02:157954] Failing at address: 0x14170000000000a0 +[a32n02:157954] [ 0] [0x1000000504d8] +[a32n02:157954] [ 1] [0x3fffea8af7e0] +[a32n02:157954] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n02:157954] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n01:128006] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n01:128006] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n03:153851] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n03:153851] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n03:153851] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n03:153851] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n02:119532] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n02:119532] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n06:60151] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n06:60151] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n06:60151] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n17:15875] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n09:122070] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n09:122070] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n09:122070] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n05:138198] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n05:138198] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n05:138198] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n05:176912] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n05:176912] *** End of error message *** +[b19n05:131264] [ 0] [0x1000000504d8] +[b19n05:131264] [ 1] [0x3fffd1cee200] +[b19n05:131264] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n05:131264] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n05:131264] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n12:178719] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n12:178719] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n12:178719] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n12:178719] *** End of error message *** +[b04n14:08953] *** Process received signal *** +[b04n14:08953] Signal: Segmentation fault (11) +[b04n14:08953] Signal code: (3) +[b04n14:08953] Failing at address: 0x14170000000000a0 +[b04n14:08953] [ 0] [0x1000000504d8] +[b04n14:08953] [ 1] [0x3fffde2286f0] +[b04n14:08953] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n14:08953] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n08:116173] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n08:116173] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n05:173083] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n05:173083] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n05:173083] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n05:173083] [10] [b01n09:173751] *** Process received signal *** +[b01n09:173751] Signal: Segmentation fault (11) +[b01n09:173751] Signal code: (3) +[b01n09:173751] Failing at address: 0xb170000000000a0 +[b01n09:173751] [ 0] [0x1000000504d8] +[b01n09:173751] [ 1] [0x3fffca27ad20] +[b01n09:173751] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n09:173751] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n09:173751] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n13:155849] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n13:155849] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n13:155849] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n13:155849] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n13:155849] [ 9] [b03n03:06826] *** Process received signal *** +[b03n03:06826] Signal: Segmentation fault (11) +[b03n03:06826] Signal code: (3) +[b03n03:06826] Failing at address: 0x14170000000000a0 +[b03n03:06826] [ 0] [0x1000000504d8] +[b03n03:06826] [ 1] [0x3fffe036d270] +[b03n03:06826] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n03:06826] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n16:116527] *** End of error message *** +[b13n15:139751] *** Process received signal *** +[b13n15:139751] Signal: Segmentation fault (11) +[b13n15:139751] Signal code: (3) +[b13n15:139751] Failing at address: 0xb170000000000a0 +[b13n15:139751] [ 0] [0x1000000504d8] +[b13n15:139751] [ 1] [0x3fffc89e39d0] +[b13n15:139751] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n15:139751] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n15:139751] [ 4] [b15n13:123710] *** Process received signal *** +[b15n13:123710] Signal: Segmentation fault (11) +[b15n13:123710] Signal code: (3) +[b15n13:123710] Failing at address: 0xb170000000000a0 +[b15n13:123710] [ 0] [0x1000000504d8] +[b15n13:123710] [ 1] [0x3fffeab3e350] +[b15n13:123710] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n13:123710] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n13:123710] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n05:135121] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n05:135121] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n05:135121] [ 6] [b22n11:120243] *** Process received signal *** +[b22n11:120243] Signal: Segmentation fault (11) +[b22n11:120243] Signal code: (3) +[b22n11:120243] Failing at address: 0x14170000000000a0 +[b22n11:120243] [ 0] [0x1000000504d8] +[b22n11:120243] [ 1] [0x3fffcab12c00] +[b22n11:120243] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n11:120243] [ 3] [a27n03:46692] *** Process received signal *** +[a27n03:46692] Signal: Segmentation fault (11) +[a27n03:46692] Signal code: (3) +[a27n03:46692] Failing at address: 0x14170000000000a0 +[a27n03:46692] [ 0] [0x1000000504d8] +[a27n03:46692] [ 1] [0x3ffff3680300] +[a27n03:46692] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n03:46692] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n08:163098] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n08:163098] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n08:163098] [11] [b15n07:124350] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n16:08387] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n16:08387] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n16:08387] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n16:08387] [ 8] [a36n12:141261] *** Process received signal *** +[a36n12:141261] Signal: Segmentation fault (11) +[a36n12:141261] Signal code: (3) +[a36n12:141261] Failing at address: 0x14170000000000a0 +[a36n12:141261] [ 0] [0x1000000504d8] +[a36n12:141261] [ 1] [0x3ffff1d7a880] +[a36n12:141261] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n12:141261] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n12:141261] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n06:43020] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n06:43020] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n06:43020] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n06:43020] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n06:43020] [ 8] [a35n03:135166] *** Process received signal *** +[a35n03:135166] Signal: Segmentation fault (11) +[a35n03:135166] Signal code: (3) +[a35n03:135166] Failing at address: 0x14170000000000a0 +[b07n04:164048] *** Process received signal *** +[b07n04:164048] Signal: Segmentation fault (11) +[b07n04:164048] Signal code: (3) +[b07n04:164048] Failing at address: 0xb170000000000a0 +[b07n04:164048] [ 0] [0x1000000504d8] +[b07n04:164048] [ 1] [0x3ffffb14edb0] +[b07n04:164048] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n04:164048] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n04:164048] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n04:126346] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n04:126346] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n04:126346] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n04:126346] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n07:172878] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n06:167369] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n06:167369] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n06:167369] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n01:145735] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n01:145735] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n02:125375] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n02:125375] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n02:125375] [ 6] [b23n08:116904] *** Process received signal *** +[b23n08:116904] Signal: Segmentation fault (11) +[b23n08:116904] Signal code: (3) +[b23n08:116904] Failing at address: 0xb170000000000a0 +[b23n08:116904] [ 0] [0x1000000504d8] +[b23n08:116904] [ 1] [0x3fffd7c03b60] +[b23n08:116904] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n08:116904] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n08:116904] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n15:154387] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n15:154387] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n18:69157] *** End of error message *** +[b06n06:163679] *** Process received signal *** +[b06n06:163679] Signal: Segmentation fault (11) +[b06n06:163679] Signal code: (3) +[b06n06:163679] Failing at address: 0x14170000000000a0 +[b06n06:163679] [ 0] [0x1000000504d8] +[b06n06:163679] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n16:130378] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n16:130378] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n16:130378] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n16:130378] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n16:130378] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n16:130378] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n03:158657] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n03:158657] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n03:158657] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n15:148581] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n15:148581] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n18:126905] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n18:126905] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n18:126905] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n18:126905] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n01:146657] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n01:146657] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n01:146657] [ 6] [b22n06:133057] *** Process received signal *** +[b22n06:133057] Signal: Segmentation fault (11) +[b22n06:133057] Signal code: (3) +[b22n06:133057] Failing at address: 0x14170000000000a0 +[b22n06:133057] [ 0] [0x1000000504d8] +[b22n06:133057] [ 1] [0x3fffd5d70fb0] +[b22n06:133057] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n06:133057] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n06:133057] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n06:164018] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n06:164018] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n06:164018] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n06:164018] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n06:164018] [13] [b18n18:121308] *** Process received signal *** +[b18n18:121308] Signal: Segmentation fault (11) +[b18n18:121308] Signal code: (3) +[b18n18:121308] Failing at address: 0xb170000000000a0 +[b18n18:121308] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n02:177550] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n02:177550] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n02:177550] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n02:177550] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n14:133634] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n14:133634] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n14:133634] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n14:133634] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n12:140122] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n12:140122] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n12:140122] [ 6] [b20n18:123414] *** Process received signal *** +[b20n18:123414] Signal: Segmentation fault (11) +[b20n18:123414] Signal code: (3) +[b20n18:123414] Failing at address: 0xb170000000000a0 +[b20n18:123414] [ 0] [0x1000000504d8] +[b20n18:123414] [ 1] [0x3ffff1dc3a80] +[b20n18:123414] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n18:123414] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n18:123414] [ 4] [b09n06:167091] *** Process received signal *** +[b09n06:167091] Signal: Segmentation fault (11) +[b09n06:167091] Signal code: (3) +[b09n06:167091] Failing at address: 0x14170000000000a0 +[b09n06:167091] [ 0] [0x1000000504d8] +[b09n06:167091] [ 1] [0x3fffe1fde190] +[b09n06:167091] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n06:167091] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n06:167091] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n07:119637] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n07:119637] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n07:119637] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n07:119637] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n07:119637] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n04:153967] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n04:153967] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n04:153967] *** End of error message *** +[b11n13:153680] *** Process received signal *** +[b11n13:153680] Signal: Segmentation fault (11) +[b11n13:153680] Signal code: (3) +[b11n13:153680] Failing at address: 0xb170000000000a0 +[b11n13:153680] [ 0] [0x1000000504d8] +[b11n13:153680] [ 1] [0x3fffcc55fd70] +[b11n13:153680] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n13:153680] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n13:153680] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n02:162036] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n02:162036] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n02:162036] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n02:162036] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n03:155730] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n03:155730] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n03:155730] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n03:155730] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n03:155730] *** End of error message *** +[0x3fffdd196780] +[a36n02:147883] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n02:147883] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n02:147883] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n02:147883] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n02:147883] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n13:168163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n13:168163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n13:168163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n13:168163] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n15:32403] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n15:32403] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n15:32403] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n18:176813] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n18:176813] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n05:174783] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n05:174783] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n05:174783] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n05:174783] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n05:174783] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n02:157954] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n02:157954] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n13:131760] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n02:121684] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n02:121684] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n02:121684] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n02:121684] [12] [b14n16:122493] *** Process received signal *** +[b14n16:122493] Signal: Segmentation fault (11) +[b14n16:122493] Signal code: (3) +[b14n16:122493] Failing at address: 0xb170000000000a0 +[b14n16:122493] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n08:164753] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n08:164753] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n08:164753] *** End of error message *** +[b06n10:176265] *** Process received signal *** +[b06n10:176265] Signal: Segmentation fault (11) +[b06n10:176265] Signal code: (3) +[b06n10:176265] Failing at address: 0xb170000000000a0 +[b06n10:176265] [ 0] [0x1000000504d8] +[b06n10:176265] [ 1] [0x3ffffc1905e0] +[b06n10:176265] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n10:176265] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n10:176265] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n09:122070] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n09:122070] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n09:122070] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n09:122070] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n13:154232] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n13:154232] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n13:154232] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n13:154232] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n13:154232] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n07:140137] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n07:140137] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n07:140137] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n07:140137] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n05:138198] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n15:85944] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n15:85944] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n15:85944] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n15:85944] [13] [b21n15:119253] *** Process received signal *** +[b21n15:119253] Signal: Segmentation fault (11) +[b21n15:119253] Signal code: (3) +[b21n15:119253] Failing at address: 0xb170000000000a0 +[b21n15:119253] [ 0] [0x1000000504d8] +[b21n15:119253] [ 1] [0x3fffd8762d90] +[b21n15:119253] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n15:119253] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n15:119253] [ 4] [b20n14:120640] *** Process received signal *** +[b20n14:120640] Signal: Segmentation fault (11) +[b20n14:120640] Signal code: (3) +[b20n14:120640] Failing at address: 0xb170000000000a0 +[b20n14:120640] [ 0] [0x1000000504d8] +[b20n14:120640] [ 1] [0x3fffe93b20d0] +[b20n14:120640] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n14:120640] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n14:120640] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n10:142931] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n10:142931] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n10:142931] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n10:142931] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n17:164295] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n08:163162] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n08:163162] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n08:163162] [ 9] [a35n01:138005] *** Process received signal *** +[a35n01:138005] Signal: Segmentation fault (11) +[a35n01:138005] Signal code: (3) +[a35n01:138005] Failing at address: 0xb170000000000a0 +[a35n01:138005] [ 0] [0x1000000504d8] +[a35n01:138005] [ 1] [0x3fffc7004780] +[a35n01:138005] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n01:138005] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n07:149480] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n07:149480] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n07:149480] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n07:149480] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n03:06826] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n03:06826] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n03:06826] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n18:164798] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n18:164798] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n08:176681] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n08:176681] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n08:176681] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n08:176681] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n14:08953] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n14:08953] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n14:08953] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n14:08953] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n14:08953] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n01:128832] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n01:128832] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n01:128832] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n05:154187] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n05:154187] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n13:133625] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n13:133625] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n04:164048] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n04:164048] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n04:164048] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n04:164048] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n09:62482] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n09:62482] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n09:62482] [10] [b08n04:172234] *** Process received signal *** +[b08n04:172234] Signal: Segmentation fault (11) +[b08n04:172234] Signal code: (3) +[b08n04:172234] Failing at address: 0x14170000000000a0 +[b08n04:172234] [ 0] [0x1000000504d8] +[b08n04:172234] [ 1] [0x3fffd958bdf0] +[b08n04:172234] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n04:172234] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n16:08387] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n16:08387] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n16:08387] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n16:08387] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n03:123616] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n03:123616] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n03:123616] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n03:123616] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n17:169739] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n17:169739] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n17:169739] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n09:166575] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n09:166575] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n09:166575] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n09:166575] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n09:166575] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n09:166575] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n05:131264] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n10:122207] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n10:122207] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n10:122207] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n10:122207] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n10:122207] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n10:122207] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n06:163676] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n06:163676] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n06:163676] *** End of error message *** +[a26n18:69152] *** Process received signal *** +[b14n12:123560] *** Process received signal *** +[b14n12:123560] Signal: Segmentation fault (11) +[b14n12:123560] Signal code: (3) +[b14n12:123560] Failing at address: 0xb170000000000a0 +[b14n12:123560] [ 0] [0x1000000504d8] +[b14n12:123560] [ 1] [0x3ffffa7127e0] +[b14n12:123560] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n12:123560] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n05:158577] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n04:126862] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n04:126862] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n04:126862] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n04:126862] *** End of error message *** +[b03n02:06564] *** Process received signal *** +[b03n02:06564] Signal: Segmentation fault (11) +[b03n02:06564] Signal code: (3) +[b03n02:06564] Failing at address: 0x14170000000000a0 +[b03n02:06564] [ 0] [0x1000000504d8] +[b03n02:06564] [ 1] [0x3fffc1d6bfe0] +[b03n02:06564] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n02:06564] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n02:06564] [ 4] [a35n15:148168] *** Process received signal *** +[a35n15:148168] Signal: Segmentation fault (11) +[a35n15:148168] Signal code: (3) +[a35n15:148168] Failing at address: 0x5170000000000a0 +[a35n15:148168] [ 0] [0x1000000504d8] +[a35n15:148168] [ 1] [0x3fffc441a4a0] +[a35n15:148168] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n15:148168] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n15:148168] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n03:46692] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n03:46692] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n03:46692] [ 6] [b16n16:127470] *** Process received signal *** +[b16n16:127470] Signal: Segmentation fault (11) +[b16n16:127470] Signal code: (3) +[b16n16:127470] Failing at address: 0xb170000000000a0 +[b16n16:127470] [ 0] [0x1000000504d8] +[b16n16:127470] [ 1] [0x3fffe4ab51e0] +[b16n16:127470] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n16:127470] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n16:127470] [ 4] [a29n02:163027] *** Process received signal *** +[a29n02:163027] Signal: Segmentation fault (11) +[a29n02:163027] Signal code: (3) +[a29n02:163027] Failing at address: 0xb170000000000a0 +[a29n02:163027] [ 0] [0x1000000504d8] +[a29n02:163027] [ 1] [0x3fffea4ac4d0] +[a29n02:163027] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n02:163027] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n02:163027] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n13:123710] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n13:123710] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n13:123710] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n13:123710] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n13:123710] [ 9] [b22n01:122014] *** Process received signal *** +[b22n01:122014] Signal: Segmentation fault (11) +[b22n01:122014] Signal code: (3) +[b22n01:122014] Failing at address: 0x14170000000000a0 +[b22n01:122014] [ 0] [0x1000000504d8] +[b22n01:122014] [ 1] [b23n04:117353] *** Process received signal *** +[b23n04:117353] Signal: Segmentation fault (11) +[b23n04:117353] Signal code: (3) +[b23n04:117353] Failing at address: 0xb170000000000a0 +[b23n04:117353] [ 0] [0x1000000504d8] +[b23n04:117353] [ 1] [0x3ffffb61bc10] +[b23n04:117353] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n04:117353] [ 3] [b02n18:03316] *** Process received signal *** +[b02n18:03316] Signal: Segmentation fault (11) +[b02n18:03316] Signal code: (3) +[b02n18:03316] Failing at address: 0xb170000000000a0 +[b02n18:03316] [ 0] [b15n07:124350] Signal: Segmentation fault (11) +[b15n07:124350] Signal code: (3) +[b15n07:124350] Failing at address: 0xb170000000000a0 +[b15n07:124350] [ 0] [0x1000000504d8] +[b15n07:124350] [ 1] [0x3fffe2d53c60] +[b15n07:124350] [ 2] [a30n03:167780] *** Process received signal *** +[a30n03:167780] Signal: Segmentation fault (11) +[a30n03:167780] Signal code: (3) +[a30n03:167780] Failing at address: 0x14170000000000a0 +[a30n03:167780] [ 0] [0x1000000504d8] +[a30n03:167780] [ 1] [0x3fffe139d210] +[a30n03:167780] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n03:167780] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n03:167780] [ 4] [b10n15:164416] *** Process received signal *** +[b10n15:164416] Signal: Segmentation fault (11) +[b10n15:164416] Signal code: (3) +[b10n15:164416] Failing at address: 0xb170000000000a0 +[b10n15:164416] [ 0] [0x1000000504d8] +[b10n15:164416] [ 1] [0x3fffef60da70] +[b10n15:164416] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n15:164416] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n15:164416] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n06:164018] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n06:164018] *** End of error message *** +[b16n14:133637] *** Process received signal *** +[b16n14:133637] Signal: Segmentation fault (11) +[b16n14:133637] Signal code: (3) +[b16n14:133637] Failing at address: 0x5170000000000a0 +[b16n14:133637] [ 0] [0x1000000504d8] +[b16n14:133637] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n10:141235] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n10:141235] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n10:141235] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n10:141235] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n02:177550] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n03:124683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n03:124683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n03:124683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n03:124683] [12] [a27n08:54625] *** Process received signal *** +[a27n08:54625] Signal: Segmentation fault (11) +[a27n08:54625] Signal code: (3) +[a27n08:54625] Failing at address: 0xb170000000000a0 +[a27n08:54625] [ 0] [0x1000000504d8] +[a27n08:54625] [ 1] [0x3ffff7e50a10] +[a27n08:54625] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n08:54625] [ 3] [a29n04:156415] *** Process received signal *** +[a29n04:156415] Signal: Segmentation fault (11) +[a29n04:156415] Signal code: (3) +[a29n04:156415] Failing at address: 0x14170000000000a0 +[a29n04:156415] [ 0] [0x1000000504d8] +[a29n04:156415] [ 1] [0x3fffed3dd5d0] +[a29n04:156415] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n04:156415] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n04:156415] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n18:123414] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n18:123414] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n18:123414] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n18:123414] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n18:123414] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n02:139025] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n02:139025] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n15:32403] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n15:32403] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n15:32403] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n15:32403] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n12:178598] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n12:178598] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n12:178598] [11] [b18n07:125452] *** Process received signal *** +[b18n07:125452] Signal: Segmentation fault (11) +[b18n07:125452] Signal code: (3) +[b18n07:125452] Failing at address: 0x14170000000000a0 +[b18n07:125452] [ 0] [0x1000000504d8] +[b18n07:125452] [ 1] [0x3fffcd307150] +[b18n07:125452] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n07:125452] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n07:125452] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n11:120243] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n11:120243] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n11:120243] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n11:120243] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n11:120243] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n02:147886] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n02:147886] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n02:147886] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n02:147886] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n02:147886] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n02:147886] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n02:147886] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n02:147886] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n02:157954] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n02:157954] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n02:157954] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n02:157954] [ 9] [b03n09:173062] *** Process received signal *** +[b03n09:173062] Signal: Segmentation fault (11) +[b03n09:173062] Signal code: (3) +[b03n09:173062] Failing at address: 0xb170000000000a0 +[b03n09:173062] [ 0] [0x1000000504d8] +[b03n09:173062] [ 1] [0x3fffe31f2e40] +[b03n09:173062] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n09:173062] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n09:124194] *** End of error message *** +[a36n18:163108] *** Process received signal *** +[a36n18:163108] Signal: Segmentation fault (11) +[a36n18:163108] Signal code: (3) +[a36n18:163108] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n07:140137] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n07:140137] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n07:140137] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n07:140137] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n07:140137] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n15:85944] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n15:85944] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n15:119253] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n15:119253] [ 6] [a32n06:154624] *** Process received signal *** +[a32n06:154624] Signal: Segmentation fault (11) +[a32n06:154624] Signal code: (3) +[a32n06:154624] Failing at address: 0x5170000000000a0 +[a32n06:154624] [ 0] [0x1000000504d8] +[a32n06:154624] [ 1] [0x3fffeb6bd110] +[a32n06:154624] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n06:154624] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n06:154624] [ 4] [b12n04:138160] *** Process received signal *** +[b12n04:138160] Signal: Segmentation fault (11) +[b12n04:138160] Signal code: (3) +[b12n04:138160] Failing at address: 0xb170000000000a0 +[b12n04:138160] [ 0] [0x1000000504d8] +[b12n04:138160] [ 1] [0x3ffff6e5a360] +[b12n04:138160] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n04:138160] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n04:138160] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n09:173751] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n09:173751] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n09:173751] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n09:173751] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n03:06826] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n03:06826] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n03:06826] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n03:06826] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n03:06826] [11] [b09n08:176684] *** Process received signal *** +[b09n08:176684] Signal: Segmentation fault (11) +[b09n08:176684] Signal code: (3) +[b09n08:176684] Failing at address: 0x14170000000000a0 +[b09n08:176684] [b23n13:114396] *** Process received signal *** +[b23n13:114396] Signal: Segmentation fault (11) +[b23n13:114396] Signal code: (3) +[b23n13:114396] Failing at address: 0xb170000000000a0 +[b23n13:114396] [ 0] [0x1000000504d8] +[b23n13:114396] [ 1] [0x3ffffef7cc50] +[b23n13:114396] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n13:114396] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n13:114396] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n18:162424] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n18:162424] *** End of error message *** +[a32n18:138476] *** Process received signal *** +[a32n18:138476] Signal: Segmentation fault (11) +[a32n18:138476] Signal code: (3) +[a32n18:138476] Failing at address: 0xb170000000000a0 +[a32n18:138476] [ 0] [0x1000000504d8] +[a32n18:138476] [ 1] [0x3ffffc524190] +[a32n18:138476] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n18:138476] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n05:121823] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n05:121823] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n05:121823] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n05:121823] [12] [b09n05:173821] *** Process received signal *** +[b09n05:173821] Signal: Segmentation fault (11) +[b09n05:173821] Signal code: (3) +[b09n05:173821] Failing at address: 0xb170000000000a0 +[b09n05:173821] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n09:62482] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n09:62482] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n09:62482] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n09:62482] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n14:08953] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n14:08953] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n14:08953] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n02:121684] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n02:121684] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n02:121684] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n04:164048] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n04:164048] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n04:164048] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n04:164048] [12] [b07n17:162362] *** Process received signal *** +[b07n17:162362] Signal: Segmentation fault (11) +[b07n17:162362] Signal code: (3) +[b07n17:162362] Failing at address: 0x14170000000000a0 +[b07n17:162362] [ 0] [0x1000000504d8] +[b07n17:162362] [ 1] [0x3fffe97967e0] +[b07n17:162362] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n17:162362] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n17:162362] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n06:43020] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n06:43020] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n09:122070] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n09:122070] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n09:122070] [13] [b06n08:174413] *** Process received signal *** +[b06n08:174413] Signal: Segmentation fault (11) +[b06n08:174413] Signal code: (3) +[b06n08:174413] Failing at address: 0x14170000000000a0 +[b06n08:174413] [ 0] [0x1000000504d8] +[b06n08:174413] [ 1] [0x3fffed97b710] +[b06n08:174413] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n08:174413] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n08:174413] [ 4] [b04n04:179175] *** Process received signal *** +[b04n04:179175] Signal: Segmentation fault (11) +[b04n04:179175] Signal code: (3) +[b04n04:179175] Failing at address: 0x14170000000000a0 +[b04n04:179175] [ 0] [0x1000000504d8] +[b04n04:179175] [ 1] [0x3fffc951b900] +[b04n04:179175] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n04:179175] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n04:179175] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n04:179175] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n08:163098] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n08:163098] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n08:163098] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n08:163098] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n16:122496] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n16:122496] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n16:122496] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n16:122496] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n16:122496] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n13:153680] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n13:153680] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n13:153680] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n13:153680] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n13:153680] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n13:153680] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n13:153680] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n13:153680] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n13:153680] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n13:153680] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n03:125244] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n03:125244] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n06:143013] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n06:143013] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n06:143013] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n06:143013] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n03:168931] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n03:168931] [10] [a35n03:135166] [ 0] [0x1000000504d8] +[a35n03:135166] [ 1] [0x3fffef7ec920] +[a35n03:135166] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n03:135166] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n03:135166] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n16:08387] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n16:08387] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n16:08387] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n08:156373] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n08:156373] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n08:156373] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n08:156373] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n06:133057] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n06:133057] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n06:133057] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n06:133057] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n06:133057] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n05:131264] [ 6] [0x3fffd0902230] +[b06n06:163679] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n06:163679] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n06:163679] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n06:163679] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n06:163679] [ 6] [b05n17:177405] *** Process received signal *** +[b05n17:177405] Signal: Segmentation fault (11) +[b05n17:177405] Signal code: (3) +[b05n17:177405] Failing at address: 0x14170000000000a0 +[b05n17:177405] [ 0] [0x1000000504d8] +[b05n17:177405] [ 1] [0x3fffebd3c350] +[b05n17:177405] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n17:177405] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n17:177405] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n15:139751] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n15:139751] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n15:139751] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n15:139751] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n15:139751] [ 9] [a26n18:69153] *** Process received signal *** +[a26n18:69153] Signal: Segmentation fault (11) +[a26n18:69153] Signal code: (3) +[a26n18:69153] Failing at address: 0xb170000000000a0 +[a26n18:69153] [ 0] [0x1000000504d8] +[a26n18:69153] [ 1] [0x3fffee6da140] +[a26n18:69153] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n18:69153] [ 3] [b17n13:123515] *** Process received signal *** +[b17n13:123515] Signal: Segmentation fault (11) +[b17n13:123515] Signal code: (3) +[b17n13:123515] Failing at address: 0xb170000000000a0 +[b17n13:123515] [ 0] [0x1000000504d8] +[b17n13:123515] [ 1] [0x3ffff71e35f0] +[b17n13:123515] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n13:123515] [ 3] [b07n06:177005] *** Process received signal *** +[b07n06:177005] Signal: Segmentation fault (11) +[b07n06:177005] Signal code: (3) +[b07n06:177005] Failing at address: 0x5170000000000a0 +[b07n06:177005] [ 0] [0x1000000504d8] +[b07n06:177005] [ 1] [0x3fffd54308b0] +[b07n06:177005] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n06:177005] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n06:177005] [ 4] [a30n18:10722] *** Process received signal *** +[a30n18:10722] Signal: Segmentation fault (11) +[a30n18:10722] Signal code: (3) +[a30n18:10722] Failing at address: 0xb170000000000a0 +[a30n18:10722] [ 0] [0x1000000504d8] +[a30n18:10722] [ 1] [0x3fffc16b23a0] +[a30n18:10722] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n18:10722] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n18:10722] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n15:137009] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n15:137009] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n15:137009] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n15:137009] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n12:160192] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n12:160192] *** End of error message *** +[a29n07:160804] *** Process received signal *** +[a29n07:160804] Signal: Segmentation fault (11) +[a29n07:160804] Signal code: (3) +[a29n07:160804] Failing at address: 0x14170000000000a0 +[a29n07:160804] [ 0] [0x1000000504d8] +[a29n07:160804] [ 1] [0x3fffd9226100] +[a29n07:160804] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n07:160804] [ 3] [b23n12:115638] *** Process received signal *** +[b23n12:115638] Signal: Segmentation fault (11) +[b23n12:115638] Signal code: (3) +[b23n12:115638] Failing at address: 0xb170000000000a0 +[b23n12:115638] [ 0] [0x1000000504d8] +[b23n12:115638] [ 1] [0x3fffcbba5970] +[b23n12:115638] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n12:115638] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n12:115638] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n07:119637] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n07:119637] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n07:119637] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n07:119637] [13] [b12n14:139847] *** Process received signal *** +[b12n14:139847] Signal: Segmentation fault (11) +[b12n14:139847] Signal code: (3) +[b12n14:139847] Failing at address: 0x5170000000000a0 +[b12n14:139847] [ 0] [0x1000000504d8] +[b12n14:139847] [ 1] [0x3ffff54f4800] +[b12n14:139847] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n14:139847] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n02:162036] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n02:162036] [10] [b02n05:164946] *** Process received signal *** +[b02n05:164946] Signal: Segmentation fault (11) +[b02n05:164946] Signal code: (3) +[b02n05:164946] Failing at address: 0x14170000000000a0 +[a34n13:137749] *** Process received signal *** +[a34n13:137749] Signal: Segmentation fault (11) +[a34n13:137749] Signal code: (3) +[a34n13:137749] Failing at address: 0x14170000000000a0 +[a34n13:137749] [ 0] [0x1000000504d8] +[a34n13:137749] [ 1] [b07n09:163519] *** Process received signal *** +[b07n09:163519] Signal: Segmentation fault (11) +[b07n09:163519] Signal code: (3) +[b07n09:163519] Failing at address: 0x14170000000000a0 +[b07n09:163519] [ 0] [0x1000000504d8] +[b07n09:163519] [ 1] [0x3fffe6e29490] +[b07n09:163519] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n09:163519] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n09:163519] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n15:155434] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n15:155434] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n06:117527] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n06:117527] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n06:117527] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n06:117527] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n06:117527] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n06:117527] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n06:117527] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n06:117527] [12] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n14:133634] *** End of error message *** +[b11n06:164015] *** Process received signal *** +[b11n06:164015] Signal: Segmentation fault (11) +[b11n06:164015] Signal code: (3) +[b11n06:164015] Failing at address: 0xb170000000000a0 +[b11n06:164015] [ 0] [b14n14:122445] *** Process received signal *** +[b14n14:122445] Signal: Segmentation fault (11) +[b14n14:122445] Signal code: (3) +[b14n14:122445] Failing at address: 0xb170000000000a0 +[b14n14:122445] [ 0] [0x1000000504d8] +[b14n14:122445] [ 1] [0x3ffff1a906a0] +[b14n14:122445] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n14:122445] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n08:54625] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n08:54625] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n08:54625] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n08:54625] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n04:156415] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n04:156415] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n04:156415] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n04:156415] [ 8] [b01n06:171502] *** Process received signal *** +[b01n06:171502] Signal: Segmentation fault (11) +[b01n06:171502] Signal code: (3) +[b01n06:171502] Failing at address: 0xb170000000000a0 +[b01n06:171502] [ 0] [b12n13:130061] *** Process received signal *** +[b12n13:130061] Signal: Segmentation fault (11) +[b12n13:130061] Signal code: (3) +[b12n13:130061] Failing at address: 0x5170000000000a0 +[b12n13:130061] [ 0] [0x1000000504d8] +[b12n13:130061] [ 1] [0x3fffd68d1500] +[b12n13:130061] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n13:130061] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n13:130061] [ 4] [b10n06:162917] *** Process received signal *** +[b10n06:162917] Signal: Segmentation fault (11) +[b10n06:162917] Signal code: (3) +[b10n06:162917] Failing at address: 0xb170000000000a0 +[b10n06:162917] [ 0] [0x1000000504d8] +[b10n06:162917] [ 1] [0x3fffe581f590] +[b10n06:162917] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n06:162917] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n02:147883] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n02:147883] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n02:147883] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n02:147883] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n02:147883] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n02:147883] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n02:147883] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n02:157954] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n02:157954] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n02:157954] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n02:157954] [13] [a36n01:141342] *** Process received signal *** +[a36n01:141342] Signal: Segmentation fault (11) +[a36n01:141342] Signal code: (3) +[a36n01:141342] Failing at address: 0xb170000000000a0 +[a36n01:141342] [ 0] [0x1000000504d8] +[a36n01:141342] [ 1] [0x3ffff2022430] +[a36n01:141342] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n01:141342] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n05:135121] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n05:135121] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n05:135121] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n05:135121] [10] [b07n03:172651] *** Process received signal *** +[b07n03:172651] Signal: Segmentation fault (11) +[b07n03:172651] Signal code: (3) +[b07n03:172651] Failing at address: 0x14170000000000a0 +[b07n03:172651] [ 0] [0x1000000504d8] +[b07n03:172651] [ 1] [0x3fffd127fb40] +[b07n03:172651] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n03:172651] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n03:172651] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n03:167780] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n03:167780] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n03:167780] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n03:167780] [ 8] [a31n18:169360] *** Process received signal *** +[a31n18:169360] Signal: Segmentation fault (11) +[a31n18:169360] Signal code: (3) +[a31n18:169360] Failing at address: 0xb170000000000a0 +[a31n18:169360] [ 0] [0x1000000504d8] +[a31n18:169360] [ 1] [0x3fffdc9d4fb0] +[a31n18:169360] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n18:169360] [ 3] [b11n03:154297] *** Process received signal *** +[b11n03:154297] Signal: Segmentation fault (11) +[b11n03:154297] Signal code: (3) +[b11n03:154297] Failing at address: 0x14170000000000a0 +[b11n03:154297] [ 0] [0x1000000504d8] +[b11n03:154297] [ 1] [0x3ffff05c5b40] +[b11n03:154297] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n03:154297] [ 3] [b19n01:121747] *** Process received signal *** +[b19n01:121747] Signal: Segmentation fault (11) +[b19n01:121747] Signal code: (3) +[b19n01:121747] Failing at address: 0x14170000000000a0 +[b19n01:121747] [ 0] [0x1000000504d8] +[b19n01:121747] [ 1] [0x3fffe71c0c40] +[b19n01:121747] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n01:121747] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n07:140137] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n07:140137] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n09:119636] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n09:119636] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n09:119636] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n09:119636] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n15:119253] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n15:119253] [ 8] [b01n05:02910] *** Process received signal *** +[b01n05:02910] Signal: Segmentation fault (11) +[b01n05:02910] Signal code: (3) +[b01n05:02910] Failing at address: 0x14170000000000a0 +[b01n05:02910] [ 0] [0x1000000504d8] +[b01n05:02910] [ 1] [0x3fffc1d93530] +[b01n05:02910] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n05:02910] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n05:02910] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n06:154624] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n06:154624] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n06:154624] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n06:154624] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n05:173083] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n05:173083] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n05:173083] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n05:173083] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n08:163162] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n08:163162] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n08:163162] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n08:163162] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n16:130378] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n16:130378] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n16:130378] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n16:130378] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n03:46692] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n03:46692] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n03:46692] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n03:46692] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n02:163027] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n02:163027] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n02:163027] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n02:163027] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n02:163027] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n02:163027] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n02:163027] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n02:163027] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n02:163027] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n13:123710] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n13:123710] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n13:123710] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n13:123710] [13] [b08n01:171789] *** Process received signal *** +[b08n01:171789] Signal: Segmentation fault (11) +[b08n01:171789] Signal code: (3) +[b08n01:171789] Failing at address: 0x14170000000000a0 +[b08n01:171789] [ 0] [0x1000000504d8] +[b08n01:171789] [ 1] [0x3ffffe633230] +[b08n01:171789] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n01:171789] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n01:171789] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n12:140122] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n12:140122] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n12:140122] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n12:140122] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n13:155849] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n13:155849] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n13:155849] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n08:176681] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n15:148168] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n15:148168] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n15:148168] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n15:148168] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n03:06826] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n03:06826] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n03:06826] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n03:06826] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n04:138160] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n04:138160] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n04:138160] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n04:138160] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n04:138160] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n03:123616] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n03:123616] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n03:123616] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n18:138476] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n18:138476] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n18:138476] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n09:62482] *** End of error message *** +[b23n05:120232] *** Process received signal *** +[b23n05:120232] Signal: Segmentation fault (11) +[b23n05:120232] Signal code: (3) +[b23n05:120232] Failing at address: 0x14170000000000a0 +[b23n05:120232] [ 0] [0x1000000504d8] +[b23n05:120232] [ 1] [0x3fffce925560] +[b23n05:120232] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n05:120232] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n05:120232] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n14:08953] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n14:08953] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n14:08953] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n05:139911] *** Process received signal *** +[a36n05:139911] Signal: Segmentation fault (11) +[a36n05:139911] Signal code: (3) +[a36n05:139911] Failing at address: 0xb170000000000a0 +[a36n05:139911] [ 0] [0x1000000504d8] +[a36n05:139911] [ 1] [0x3fffdcfe5100] +[a36n05:139911] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n05:139911] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n13:133625] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n13:133625] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n13:133625] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n13:133625] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n04:164048] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n04:164048] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n04:164048] *** End of error message *** +[b09n13:164689] *** Process received signal *** +[b09n13:164689] Signal: Segmentation fault (11) +[b09n13:164689] Signal code: (3) +[b09n13:164689] Failing at address: 0xb170000000000a0 +[b09n13:164689] [ 0] [0x1000000504d8] +[b09n13:164689] [ 1] [0x3fffcaf23670] +[b09n13:164689] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n13:164689] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n01:138005] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n01:138005] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n01:138005] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n01:138005] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n01:138005] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n04:179175] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n04:179175] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n04:179175] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n04:179175] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n04:179175] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n06:167369] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n06:167369] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n06:167369] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n06:167369] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n09:122070] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n09:122070] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n06:167091] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n06:167091] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n06:167091] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n06:167091] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n06:167091] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n06:167091] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n06:167091] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n06:167091] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n06:167091] [13] [b16n12:134495] *** Process received signal *** +[b16n12:134495] Signal: Segmentation fault (11) +[b16n12:134495] Signal code: (3) +[b16n12:134495] Failing at address: 0x14170000000000a0 +[b16n12:134495] [ 0] [0x1000000504d8] +[b16n12:134495] [ 1] [0x3ffff71c2190] +[b16n12:134495] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n12:134495] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n12:134495] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n02:125375] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n02:125375] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n02:125375] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n02:125375] [10] [0x1000000504d8] +[b18n18:121308] [ 1] [0x3fffde0209d0] +[b18n18:121308] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n18:121308] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n18:121308] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n18:121308] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n01:128832] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n01:128832] *** End of error message *** +[0x3ffffd152f60] +[b22n01:122014] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n01:122014] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n01:122014] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n01:122014] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n01:122014] [ 6] [a26n18:69152] Signal: Segmentation fault (11) +[a26n18:69152] Signal code: (3) +[a26n18:69152] Failing at address: 0x14170000000000a0 +[a26n18:69152] [ 0] [0x1000000504d8] +[a26n18:69152] [ 1] [0x3fffd17ab9e0] +[a26n18:69152] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n18:69152] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n10:176265] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n10:176265] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n10:176265] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n10:176265] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n10:176265] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n10:176265] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n10:176265] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n10:176265] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n10:176265] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n11:120243] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n11:120243] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n11:120243] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n09:166575] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n09:166575] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n09:166575] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n09:166575] [14] [b14n13:122141] *** Process received signal *** +[b14n13:122141] Signal: Segmentation fault (11) +[b14n13:122141] Signal code: (3) +[b14n13:122141] Failing at address: 0xb170000000000a0 +[b14n13:122141] [ 0] [0x1000000504d8] +[b14n13:122141] [ 1] [0x3fffe9a022c0] +[b14n13:122141] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n13:122141] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n13:122141] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n03:158657] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n03:158657] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n16:127470] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n16:127470] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n16:127470] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n16:127470] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n16:127470] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n16:127470] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n16:127470] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n16:127470] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n16:127470] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n18:126905] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n18:126905] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n18:126905] [10] [b05n18:165698] *** Process received signal *** +[b05n18:165698] Signal: Segmentation fault (11) +[b05n18:165698] Signal code: (3) +[b05n18:165698] Failing at address: 0xb170000000000a0 +[b05n18:165698] [ 0] [0x1000000504d8] +[b05n18:165698] [ 1] [0x3fffda317020] +[b05n18:165698] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n18:165698] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n18:165698] [ 4] [b19n04:130746] *** Process received signal *** +[b19n04:130746] Signal: Segmentation fault (11) +[b19n04:130746] Signal code: (3) +[b19n04:130746] Failing at address: 0xb170000000000a0 +[b19n04:130746] [ 0] [0x1000000504d8] +[b19n04:130746] [ 1] [0x3fffddd57bf0] +[b19n04:130746] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n04:130746] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n04:130746] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n02:06564] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n02:06564] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n02:06564] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n02:06564] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n02:06564] [ 9] [a35n08:138176] *** Process received signal *** +[a35n08:138176] Signal: Segmentation fault (11) +[a35n08:138176] Signal code: (3) +[a35n08:138176] Failing at address: 0xb170000000000a0 +[a35n08:138176] [ 0] [0x1000000504d8] +[a35n08:138176] [ 1] [0x3fffc0d201a0] +[a35n08:138176] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n08:138176] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n10:126790] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n10:126790] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n10:126790] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n10:126790] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n06:163679] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n06:163679] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n06:163679] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n07:160804] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n07:160804] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n07:160804] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n07:160804] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n07:160804] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n07:160804] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n12:141261] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n12:141261] [ 6] [b16n10:124503] *** Process received signal *** +[b16n10:124503] Signal: Segmentation fault (11) +[b16n10:124503] Signal code: (3) +[b16n10:124503] Failing at address: 0x14170000000000a0 +[b16n10:124503] [ 0] [0x1000000504d8] +[b16n10:124503] [ 1] [0x3ffff5ec9140] +[b16n10:124503] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n10:124503] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n10:124503] [ 4] [a34n14:150135] *** Process received signal *** +[a34n14:150135] Signal: Segmentation fault (11) +[a34n14:150135] Signal code: (3) +[a34n14:150135] Failing at address: 0xb170000000000a0 +[a34n14:150135] [ 0] [0x1000000504d8] +[a34n14:150135] [ 1] [0x3ffff4823370] +[a34n14:150135] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n14:150135] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n14:150135] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n06:177005] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n06:177005] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n06:177005] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n06:177005] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n14:139847] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n14:139847] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n14:139847] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n14:139847] [ 7] [b14n15:123039] *** Process received signal *** +[b14n15:123039] Signal: Segmentation fault (11) +[b14n15:123039] Signal code: (3) +[b14n15:123039] Failing at address: 0xb170000000000a0 +[b14n15:123039] [ 0] [0x1000000504d8] +[b14n15:123039] [ 1] [0x3fffc9fa2580] +[b14n15:123039] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n15:123039] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n07:119637] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n07:119637] *** End of error message *** +[a32n16:147285] *** Process received signal *** +[a32n16:147285] Signal: Segmentation fault (11) +[a32n16:147285] Signal code: (3) +[a32n16:147285] Failing at address: 0xb170000000000a0 +[a32n16:147285] [ 0] [0x1000000504d8] +[a32n16:147285] [ 1] [0x3ffffdcefaf0] +[a32n16:147285] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n16:147285] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n16:147285] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n13:153680] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n04:172234] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n04:172234] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n04:172234] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n04:172234] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n09:163519] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n09:163519] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n09:163519] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n09:163519] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n09:163519] [ 9] [a35n06:138767] *** Process received signal *** +[a35n06:138767] Signal: Segmentation fault (11) +[a35n06:138767] Signal code: (3) +[a35n06:138767] Failing at address: 0x14170000000000a0 +[a35n06:138767] [ 0] [0x1000000504d8] +[a35n06:138767] [ 1] [0x3fffc8e44780] +[a35n06:138767] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n06:138767] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n06:138767] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n03:124683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n03:124683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n03:124683] *** End of error message *** +[ 0] [0x1000000504d8] +[b09n05:173821] [ 1] [0x3fffdcdb0bc0] +[b09n05:173821] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n05:173821] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n05:173821] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n05:173821] [ 5] [b04n15:177858] *** Process received signal *** +[b04n15:177858] Signal: Segmentation fault (11) +[b04n15:177858] Signal code: (3) +[b04n15:177858] Failing at address: 0xb170000000000a0 +[b04n15:177858] [ 0] [0x1000000504d8] +[b04n15:177858] [ 1] [b12n06:135852] *** Process received signal *** +[b12n06:135852] Signal: Segmentation fault (11) +[b12n06:135852] Signal code: (3) +[b12n06:135852] Failing at address: 0x14170000000000a0 +[b12n06:135852] [ 0] [0x1000000504d8] +[b12n06:135852] [ 1] [0x3fffd39a9390] +[b12n06:135852] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n06:135852] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n12:178598] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n12:178598] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n12:178598] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n12:178598] *** End of error message *** +[b02n12:180115] *** Process received signal *** +[b02n12:180115] Signal: Segmentation fault (11) +[b02n12:180115] Signal code: (3) +[b02n12:180115] Failing at address: 0xb170000000000a0 +[b02n12:180115] [ 0] [0x1000000504d8] +[b02n12:180115] [ 1] [0x3fffd2db4660] +[b02n12:180115] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n12:180115] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n12:180115] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n05:131264] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n05:131264] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n06:133057] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n06:133057] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n06:133057] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n06:133057] [13] [b10n10:126195] *** Process received signal *** +[b10n10:126195] Signal: Segmentation fault (11) +[b10n10:126195] Signal code: (3) +[b10n10:126195] Failing at address: 0xb170000000000a0 +[b10n10:126195] [ 0] [0x1000000504d8] +[b10n10:126195] [ 1] [0x3ffffe952a10] +[b10n10:126195] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n10:126195] [ 3] [b03n13:01303] *** Process received signal *** +[b03n13:01303] Signal: Segmentation fault (11) +[b03n13:01303] Signal code: (3) +[b03n13:01303] Failing at address: 0x14170000000000a0 +[b03n13:01303] [ 0] [0x1000000504d8] +[b03n13:01303] [ 1] [0x3fffd47047a0] +[b03n13:01303] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n13:01303] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n13:01303] [ 4] [0x3fffdf6cb910] +[b16n14:133637] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n14:133637] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n14:133637] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n14:133637] [ 5] [0x1000000504d8] +[b11n06:164015] [ 1] [0x3fffee453ac0] +[b11n06:164015] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n06:164015] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n06:164015] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n06:164015] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n08:54625] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n08:54625] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n08:54625] [10] [b04n18:176811] *** Process received signal *** +[b04n18:176811] Signal: Segmentation fault (11) +[b04n18:176811] Signal code: (3) +[b04n18:176811] Failing at address: 0x5170000000000a0 +[b04n18:176811] [ 0] [0x1000000504d8] +[b04n18:176811] [ 1] [0x3fffea306bd0] +[b04n18:176811] [ 2] [b19n09:123420] *** Process received signal *** +[b19n09:123420] Signal: Segmentation fault (11) +[b19n09:123420] Signal code: (3) +[b19n09:123420] Failing at address: 0x14170000000000a0 +[b19n09:123420] [ 0] [0x1000000504d8] +[b19n09:123420] [ 1] [0x3fffcc0d8540] +[b19n09:123420] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n09:123420] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n01:146657] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n01:146657] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n01:146657] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n01:146657] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n02:147886] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n02:147886] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n02:147886] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n02:147886] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n02:157954] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n02:157954] *** End of error message *** +[a35n07:138203] *** Process received signal *** +[a35n07:138203] Signal: Segmentation fault (11) +[a35n07:138203] Signal code: (3) +[a35n07:138203] Failing at address: 0xb170000000000a0 +[a35n07:138203] [ 0] [0x1000000504d8] +[a35n07:138203] [ 1] [0x3fffcce68290] +[a35n07:138203] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n07:138203] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n07:138203] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n15:139751] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n15:139751] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n15:139751] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n15:139751] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n13:130061] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n13:130061] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n13:130061] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n13:130061] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n10:122207] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n10:122207] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n10:122207] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n10:122207] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n10:122207] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n10:122207] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n09:173751] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n09:173751] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n09:173751] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n09:173751] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n09:173751] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n03:167780] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n03:167780] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n03:167780] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n03:167780] [12] [b06n08:174410] *** Process received signal *** +[b06n08:174410] Signal: Segmentation fault (11) +[b06n08:174410] Signal code: (3) +[b06n08:174410] Failing at address: 0x5170000000000a0 +[b06n08:174410] [ 0] [0x1000000504d8] +[b06n08:174410] [ 1] [0x3ffff0d27ee0] +[b06n08:174410] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n08:174410] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n08:174410] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n03:154297] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n03:154297] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n03:154297] [ 6] [b04n07:02842] *** Process received signal *** +[b04n07:02842] Signal: Segmentation fault (11) +[b04n07:02842] Signal code: (3) +[b04n07:02842] Failing at address: 0x5170000000000a0 +[b04n07:02842] [ 0] [0x1000000504d8] +[b04n07:02842] [ 1] [0x3fffda5aa660] +[b04n07:02842] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n07:02842] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n07:02842] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n15:119253] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n15:119253] [10] [0x1000000504d8] +[b02n18:03316] [ 1] [0x3fffc7485cd0] +[b02n18:03316] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n18:03316] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n18:03316] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n18:03316] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n18:03316] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n18:03316] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n08:116904] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n08:116904] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n08:116904] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n08:116904] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n08:116904] [ 9] [0x1000000504d8] +[b14n16:122493] [ 1] [0x3fffe89ea270] +[b14n16:122493] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n16:122493] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n16:122493] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n16:122493] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n06:154624] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n06:154624] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n06:154624] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n06:154624] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n03:46692] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n03:46692] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n03:46692] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n03:46692] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n13:123710] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n13:123710] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n05:121823] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n05:121823] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n05:121823] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n12:168646] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n12:168646] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n12:168646] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n12:168646] *** End of error message *** +[a28n05:04522] *** Process received signal *** +[a28n05:04522] Signal: Segmentation fault (11) +[a28n05:04522] Signal code: (3) +[a28n05:04522] Failing at address: 0x14170000000000a0 +[a28n05:04522] [ 0] [0x1000000504d8] +[a28n05:04522] [ 1] [0x3fffe23e8860] +[a28n05:04522] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n05:04522] [ 3] [a32n13:163196] *** Process received signal *** +[a32n13:163196] Signal: Segmentation fault (11) +[a32n13:163196] Signal code: (3) +[a32n13:163196] Failing at address: 0x14170000000000a0 +[a32n13:163196] [ 0] [0x1000000504d8] +[a32n13:163196] [ 1] [0x3fffdb079250] +[a32n13:163196] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n13:163196] [ 3] [ 0] [0x1000000504d8] +[b09n08:176684] [ 1] [0x3fffdeb93080] +[b09n08:176684] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n08:176684] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n08:176684] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n08:176684] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n08:176684] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n08:176684] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n08:176684] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n08:176684] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n08:176684] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n08:176684] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n08:176684] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n08:176684] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n08:176684] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n08:176684] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n04:156415] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n04:156415] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n04:156415] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n12:140122] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n12:140122] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n12:140122] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n10:142931] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n10:142931] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n06:43020] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n06:43020] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n06:43020] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n06:43020] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n06:43020] *** End of error message *** +[b09n17:163179] *** Process received signal *** +[b09n17:163179] Signal: Segmentation fault (11) +[b09n17:163179] Signal code: (3) +[b09n17:163179] Failing at address: 0xb170000000000a0 +[b09n17:163179] [ 0] [0x1000000504d8] +[b09n17:163179] [ 1] [0x3fffea514e10] +[b09n17:163179] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n17:163179] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n17:163179] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n17:163179] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n18:138476] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n18:138476] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n18:138476] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n10:141235] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n10:141235] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n05:158577] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n05:158577] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n05:158577] [ 9] [b04n14:08953] *** End of error message *** +[b21n08:116176] *** Process received signal *** +[b21n08:116176] Signal: Segmentation fault (11) +[b21n08:116176] Signal code: (3) +[b21n08:116176] Failing at address: 0x14170000000000a0 +[b21n08:116176] [ 0] [0x1000000504d8] +[b21n08:116176] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n04:179175] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n04:179175] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n04:179175] [13] [a27n04:52708] *** Process received signal *** +[a27n04:52708] Signal: Segmentation fault (11) +[a27n04:52708] Signal code: (3) +[a27n04:52708] Failing at address: 0x14170000000000a0 +[a27n04:52708] [ 0] [0x1000000504d8] +[a27n04:52708] [ 1] [0x3fffda36ebd0] +[a27n04:52708] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n04:52708] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n04:52708] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n03:10293] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n03:10293] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n03:10293] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n03:10293] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n03:10293] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n03:10293] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n03:10293] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n03:10293] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n03:10293] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n17:162362] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n17:162362] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n17:162362] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n17:162362] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n13:133625] *** End of error message *** +[b16n04:123056] *** Process received signal *** +[b16n04:123056] Signal: Segmentation fault (11) +[b16n04:123056] Signal code: (3) +[b16n04:123056] Failing at address: 0x14170000000000a0 +[b16n04:123056] [ 0] [0x1000000504d8] +[b16n04:123056] [ 1] [0x3fffc14b4480] +[b16n04:123056] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n04:123056] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n04:123056] [ 4] [b15n03:119832] *** Process received signal *** +[b15n03:119832] Signal: Segmentation fault (11) +[b15n03:119832] Signal code: (3) +[b15n03:119832] Failing at address: 0x14170000000000a0 +[b15n03:119832] [ 0] [0x1000000504d8] +[b15n03:119832] [ 1] [0x3fffec1ff480] +[b15n03:119832] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n03:119832] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n14:120640] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n14:120640] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n14:120640] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n18:69153] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n18:69153] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n18:69153] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n18:164798] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n18:164798] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n18:164798] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n18:164798] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n11:120243] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n11:120243] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n11:120243] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n11:120243] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n03:172651] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n03:172651] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n03:172651] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n03:172651] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n07:125452] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n07:125452] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n07:125452] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n07:125452] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n07:125452] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n07:125452] [10] [b21n07:117360] *** Process received signal *** +[b21n07:117360] Signal: Segmentation fault (11) +[b21n07:117360] Signal code: (3) +[b21n07:117360] Failing at address: 0xb170000000000a0 +[b21n07:117360] [ 0] [0x1000000504d8] +[b21n07:117360] [ 1] [0x3fffd7c71fe0] +[b21n07:117360] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n07:117360] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n12:123560] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n12:123560] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n12:123560] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n12:123560] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n12:123560] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n04:138160] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n04:138160] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n04:138160] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n04:138160] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n02:163027] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n02:163027] *** End of error message *** +[a26n11:62423] *** Process received signal *** +[a26n11:62423] Signal: Segmentation fault (11) +[a26n11:62423] Signal code: (3) +[a26n11:62423] Failing at address: 0xb170000000000a0 +[a26n11:62423] [ 0] [0x1000000504d8] +[a26n11:62423] [ 1] [0x3fffdd195820] +[a26n11:62423] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n11:62423] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n11:62423] [ 4] [b19n07:131674] *** Process received signal *** +[b19n07:131674] Signal: Segmentation fault (11) +[b19n07:131674] Signal code: (3) +[b19n07:131674] Failing at address: 0x14170000000000a0 +[b19n07:131674] [ 0] [0x1000000504d8] +[b19n07:131674] [ 1] [0x3fffd8c607f0] +[b19n07:131674] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n07:131674] [ 3] [a30n15:154384] *** Process received signal *** +[a30n15:154384] Signal: Segmentation fault (11) +[a30n15:154384] Signal code: (3) +[a30n15:154384] Failing at address: 0xb170000000000a0 +[a30n15:154384] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n18:169360] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n18:169360] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n18:169360] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n18:169360] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n18:169360] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n18:169360] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n18:169360] [10] [b02n05:164946] [ 0] [0x1000000504d8] +[b02n05:164946] [ 1] [0x3fffca6be3c0] +[b02n05:164946] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n05:164946] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n05:164946] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n05:164946] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n05:164946] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n05:164946] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n05:164946] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n05:164946] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n17:177405] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n17:177405] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n17:177405] [ 7] [b15n04:133905] *** Process received signal *** +[b15n04:133905] Signal: Segmentation fault (11) +[b15n04:133905] Signal code: (3) +[b15n04:133905] Failing at address: 0xb170000000000a0 +[b15n04:133905] [ 0] [0x1000000504d8] +[b15n04:133905] [ 1] [0x3fffedfbc730] +[b15n04:133905] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n04:133905] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n04:133905] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n08:156373] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n08:156373] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n08:156373] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n08:156373] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n16:130378] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n16:130378] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n14:139847] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n14:139847] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n14:139847] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n06:177005] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n06:177005] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n06:177005] [11] [a33n05:139435] *** Process received signal *** +[a33n05:139435] Signal: Segmentation fault (11) +[a33n05:139435] Signal code: (3) +[a33n05:139435] Failing at address: 0xb170000000000a0 +[a33n05:139435] [ 0] [0x1000000504d8] +[a33n05:139435] [ 1] [0x3fffee757810] +[a33n05:139435] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n05:139435] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n05:139435] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n05:139435] [ 5] [a28n04:13084] *** Process received signal *** +[a28n04:13084] Signal: Segmentation fault (11) +[a28n04:13084] Signal code: (3) +[a28n04:13084] Failing at address: 0x14170000000000a0 +[a28n04:13084] [ 0] [0x1000000504d8] +[a28n04:13084] [ 1] [0x3fffe5930ba0] +[a28n04:13084] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n04:13084] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n04:13084] [ 4] [a29n06:157792] *** Process received signal *** +[a29n06:157792] Signal: Segmentation fault (11) +[a29n06:157792] Signal code: (3) +[a29n06:157792] Failing at address: 0x14170000000000a0 +[a29n06:157792] [ 0] [0x1000000504d8] +[a29n06:157792] [ 1] [0x3fffee962ff0] +[a29n06:157792] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n06:157792] [ 3] [b08n16:171917] *** Process received signal *** +[b08n16:171917] Signal: Segmentation fault (11) +[b08n16:171917] Signal code: (3) +[b08n16:171917] Failing at address: 0xb170000000000a0 +[b08n16:171917] [ 0] [b08n10:11493] *** Process received signal *** +[b08n10:11493] Signal: Segmentation fault (11) +[b08n10:11493] Signal code: (3) +[b08n10:11493] Failing at address: 0x14170000000000a0 +[b08n10:11493] [ 0] [0x1000000504d8] +[b08n10:11493] [ 1] [0x3fffeba97ca0] +[b08n10:11493] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n10:11493] [ 3] [b22n10:118388] *** Process received signal *** +[b06n02:163323] *** Process received signal *** +[b06n02:163323] Signal: Segmentation fault (11) +[b06n02:163323] Signal code: (3) +[b06n02:163323] Failing at address: 0x14170000000000a0 +[b06n02:163323] [ 0] [0x1000000504d8] +[b06n02:163323] [ 1] [0x3fffdbdd4860] +[b06n02:163323] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n02:163323] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n02:163323] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n06:135852] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n06:135852] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n06:135852] [ 6] [0x3fffecbb5930] +[b04n15:177858] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n15:177858] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n15:177858] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n15:177858] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n04:117353] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n04:117353] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n04:117353] [ 6] [b21n09:120697] *** Process received signal *** +[b21n09:120697] Signal: Segmentation fault (11) +[b21n09:120697] Signal code: (3) +[b21n09:120697] Failing at address: 0x14170000000000a0 +[b21n09:120697] [ 0] [0x1000000504d8] +[b21n09:120697] [ 1] [0x3ffff8a91e50] +[b21n09:120697] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n09:120697] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n09:120697] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n03:135166] [ 5] [b14n03:130347] *** Process received signal *** +[b14n03:130347] Signal: Segmentation fault (11) +[b14n03:130347] Signal code: (3) +[b14n03:130347] Failing at address: 0x14170000000000a0 +[b14n03:130347] [ 0] [0x1000000504d8] +[b14n03:130347] [ 1] [0x3fffd15a7ba0] +[b14n03:130347] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n03:130347] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n03:130347] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n02:162036] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n02:162036] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n02:162036] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n02:162036] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n02:162036] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n14:133637] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n14:133637] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n14:133637] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n14:133637] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n08:54625] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n08:54625] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n08:54625] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n08:54625] [14] [a31n09:164365] *** Process received signal *** +[a31n09:164365] Signal: Segmentation fault (11) +[a31n09:164365] Signal code: (3) +[a31n09:164365] Failing at address: 0xb170000000000a0 +[a31n09:164365] [ 0] [0x1000000504d8] +[a31n09:164365] [ 1] [0x3fffc040ab80] +[a31n09:164365] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n09:164365] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n09:164365] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n06:164015] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n06:164015] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n06:164015] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n06:164015] [ 9] [b04n18:176810] *** Process received signal *** +[b04n18:176810] Signal: Segmentation fault (11) +[b04n18:176810] Signal code: (3) +[b04n18:176810] Failing at address: 0xb170000000000a0 +[b23n13:114399] *** Process received signal *** +[b23n13:114399] Signal: Segmentation fault (11) +[b23n13:114399] Signal code: (3) +[b23n13:114399] Failing at address: 0x14170000000000a0 +[b23n13:114399] [ 0] [0x1000000504d8] +[b23n13:114399] [ 1] [0x3fffea893dc0] +[b23n13:114399] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n13:114399] [ 3] [a36n01:141345] *** Process received signal *** +[a36n01:141345] Signal: Segmentation fault (11) +[a36n01:141345] Signal code: (3) +[a36n01:141345] Failing at address: 0x5170000000000a0 +[a36n01:141345] [ 0] [0x1000000504d8] +[a36n01:141345] [ 1] [0x3fffd7599770] +[a36n01:141345] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n01:141345] [ 3] [a33n11:156156] *** Process received signal *** +[a33n11:156156] Signal: Segmentation fault (11) +[a33n11:156156] Signal code: (3) +[a33n11:156156] Failing at address: 0x14170000000000a0 +[a33n11:156156] [ 0] [0x1000000504d8] +[a33n11:156156] [ 1] [0x3ffff699d610] +[a33n11:156156] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n11:156156] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n11:156156] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n07:160804] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n07:160804] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n07:160804] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n07:160804] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n07:160804] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n07:160804] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n13:164689] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n13:164689] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n13:164689] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n07:124350] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n07:124350] [ 4] [a36n18:163108] [ 0] [0x1000000504d8] +[a36n18:163108] [ 1] [0x3fffc4909350] +[a36n18:163108] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n18:163108] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n18:163108] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n18:163108] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n18:163108] [ 6] [b02n14:169525] *** Process received signal *** +[b02n14:169525] Signal: Segmentation fault (11) +[b02n14:169525] Signal code: (3) +[b02n14:169525] Failing at address: 0x14170000000000a0 +[b02n14:169525] [ 0] [0x1000000504d8] +[b02n14:169525] [ 1] [0x3fffe390bbe0] +[b02n14:169525] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n14:169525] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n14:169525] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n12:134495] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n12:134495] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n12:134495] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n12:134495] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n12:134495] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n12:134495] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n12:134495] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n12:134495] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n12:134495] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n13:01303] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n13:01303] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n13:01303] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n13:01303] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n13:01303] [ 9] [b10n07:116116] *** Process received signal *** +[b10n07:116116] Signal: Segmentation fault (11) +[b10n07:116116] Signal code: (3) +[b10n07:116116] Failing at address: 0x14170000000000a0 +[b10n07:116116] [ 0] [0x1000000504d8] +[b10n07:116116] [ 1] [0x3fffece8d150] +[b10n07:116116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n07:116116] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n07:116116] [ 4] [b21n11:120582] *** Process received signal *** +[b21n11:120582] Signal: Segmentation fault (11) +[b21n11:120582] Signal code: (3) +[b21n11:120582] Failing at address: 0xb170000000000a0 +[b21n11:120582] [ 0] [0x1000000504d8] +[b21n11:120582] [ 1] [0x3ffff46dd100] +[b21n11:120582] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n11:120582] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n13:123515] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n13:123515] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n13:123515] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n03:125244] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n03:125244] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n03:125244] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n03:125244] [ 9] [b10n02:164490] *** Process received signal *** +[b10n02:164490] Signal: Segmentation fault (11) +[b10n02:164490] Signal code: (3) +[b10n02:164490] Failing at address: 0xb170000000000a0 +[b10n02:164490] [ 0] [0x1000000504d8] +[b10n02:164490] [ 1] [0x3fffc9066130] +[b10n02:164490] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n02:164490] [ 3] [a34n01:164965] *** Process received signal *** +[a34n01:164965] Signal: Segmentation fault (11) +[a34n01:164965] Signal code: (3) +[a34n01:164965] Failing at address: 0x14170000000000a0 +[a34n01:164965] [ 0] [0x1000000504d8] +[a34n01:164965] [ 1] [0x3ffff80074c0] +[a34n01:164965] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n01:164965] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n18:10722] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n18:10722] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n18:10722] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n18:10722] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n18:10722] [ 9] [a33n10:140417] *** Process received signal *** +[a33n10:140417] Signal: Segmentation fault (11) +[a33n10:140417] Signal code: (3) +[a33n10:140417] Failing at address: 0x14170000000000a0 +[a33n10:140417] [ 0] [0x1000000504d8] +[a33n10:140417] [ 1] [0x3fffe2d72b50] +[a33n10:140417] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n10:140417] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n10:140417] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n06:167091] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n06:167091] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n12:132101] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n01:171789] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n01:171789] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n01:171789] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n01:171789] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n01:171789] [ 9] [b23n02:116889] *** Process received signal *** +[b23n02:116889] Signal: Segmentation fault (11) +[b23n02:116889] Signal code: (3) +[b23n02:116889] Failing at address: 0x14170000000000a0 +[b23n02:116889] [ 0] [0x1000000504d8] +[b23n02:116889] [ 1] [0x3fffe7431b50] +[b23n02:116889] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n02:116889] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n02:116889] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n02:116889] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n03:167780] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n03:167780] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n03:167780] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n10:176265] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n10:176265] *** End of error message *** +[b19n01:121744] *** Process received signal *** +[b19n01:121744] Signal: Segmentation fault (11) +[b19n01:121744] Signal code: (3) +[b19n01:121744] Failing at address: 0xb170000000000a0 +[b19n01:121744] [ 0] [0x1000000504d8] +[b19n01:121744] [ 1] [0x3fffcbc4b060] +[b19n01:121744] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n01:121744] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n03:154297] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n03:154297] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n03:154297] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n15:119253] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n15:119253] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n15:119253] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n15:119253] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n16:127470] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n16:127470] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n07:02842] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n07:02842] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n07:02842] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n07:02842] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n06:154624] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n06:154624] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n06:154624] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n03:46692] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n18:123414] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n18:123414] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n18:123414] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n18:123414] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n04:156415] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n04:156415] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n04:156415] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n04:156415] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n05:04522] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n05:04522] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n05:04522] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n18:138476] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n18:138476] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n18:138476] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n18:138476] [13] [a29n17:169736] *** Process received signal *** +[a29n17:169736] Signal: Segmentation fault (11) +[a29n17:169736] Signal code: (3) +[a29n17:169736] Failing at address: 0xb170000000000a0 +[a29n17:169736] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n15:148168] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n15:148168] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n15:148168] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n04:179175] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n04:179175] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n04:52708] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n04:52708] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n04:52708] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n04:52708] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n04:52708] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n04:52708] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n04:52708] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n04:52708] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n04:52708] [13] [b13n15:139748] *** Process received signal *** +[b13n15:139748] Signal: Segmentation fault (11) +[b13n15:139748] Signal code: (3) +[b13n15:139748] Failing at address: 0xb170000000000a0 +[b13n15:139748] [ 0] [0x1000000504d8] +[b13n15:139748] [ 1] [0x3fffc4386420] +[b13n15:139748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n15:139748] [ 3] [b12n12:138736] *** Process received signal *** +[b12n12:138736] Signal: Segmentation fault (11) +[b12n12:138736] Signal code: (3) +[b12n12:138736] Failing at address: 0xb170000000000a0 +[b12n12:138736] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n04:123056] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n04:123056] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n04:123056] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n04:123056] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n04:123056] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n09:166575] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n06:143013] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n06:143013] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n06:143013] [11] [b22n05:123935] *** Process received signal *** +[b22n05:123935] Signal: Segmentation fault (11) +[b22n05:123935] Signal code: (3) +[b22n05:123935] Failing at address: 0xb170000000000a0 +[b22n05:123935] [ 0] [0x1000000504d8] +[b22n05:123935] [ 1] [0x3fffdb3ebcd0] +[b22n05:123935] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n05:123935] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n05:123935] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n06:163679] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n06:163679] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n06:163679] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n06:163679] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n03:123616] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n03:123616] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n03:123616] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n02:06564] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n02:06564] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n02:06564] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n02:06564] [13] [b04n06:10227] *** Process received signal *** +[b04n06:10227] Signal: Segmentation fault (11) +[b04n06:10227] Signal code: (3) +[b04n06:10227] Failing at address: 0x5170000000000a0 +[b04n06:10227] [ 0] [0x1000000504d8] +[b04n06:10227] [ 1] [0x3fffe75c4c00] +[b04n06:10227] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n06:10227] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n06:10227] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n10:124503] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n10:124503] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n10:124503] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n10:124503] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n10:124503] [ 9] [b08n08:164750] *** Process received signal *** +[b08n08:164750] Signal: Segmentation fault (11) +[b08n08:164750] Signal code: (3) +[b08n08:164750] Failing at address: 0xb170000000000a0 +[b08n08:164750] [ 0] [0x1000000504d8] +[b08n08:164750] [ 1] [0x3fffc3808b30] +[b08n08:164750] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n08:164750] [ 3] [b21n10:130584] *** Process received signal *** +[b21n10:130584] Signal: Segmentation fault (11) +[b21n10:130584] Signal code: (3) +[b21n10:130584] Failing at address: 0x14170000000000a0 +[b21n10:130584] [ 0] [0x1000000504d8] +[b21n10:130584] [ 1] [0x3fffe3e190a0] +[b21n10:130584] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n10:130584] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n15:137009] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n15:137009] *** End of error message *** +[a32n15:139059] *** Process received signal *** +[a32n15:139059] Signal: Segmentation fault (11) +[a32n15:139059] Signal code: (3) +[a32n15:139059] Failing at address: 0x14170000000000a0 +[a32n15:139059] [ 0] [0x1000000504d8] +[a32n15:139059] [ 1] [0x3ffffc0a9f10] +[a32n15:139059] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n15:139059] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n15:139059] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n13:155849] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n13:155849] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n13:155849] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n03:172651] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n03:172651] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n03:172651] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n03:172651] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n12:115638] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n12:115638] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n12:115638] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n12:115638] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n02:147883] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n02:147883] *** End of error message *** +[a29n09:79008] *** Process received signal *** +[a29n09:79008] Signal: Segmentation fault (11) +[a29n09:79008] Signal code: (3) +[a29n09:79008] Failing at address: 0xb170000000000a0 +[a29n09:79008] [ 0] [0x1000000504d8] +[a29n09:79008] [ 1] [0x3fffeadbff10] +[a29n09:79008] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n09:79008] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n12:140122] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n12:140122] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n14:122445] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n14:122445] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n14:122445] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n14:122445] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n14:122445] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n14:122445] [ 9] [b22n14:116412] *** Process received signal *** +[b22n14:116412] Signal: Segmentation fault (11) +[b22n14:116412] Signal code: (3) +[b22n14:116412] Failing at address: 0x14170000000000a0 +[b22n14:116412] [ 0] [0x1000000504d8] +[b22n14:116412] [ 1] [0x3fffc0dc3870] +[b22n14:116412] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n14:116412] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n18:165698] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n18:165698] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n18:165698] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n18:165698] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n18:165698] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n18:165698] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n18:165698] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n18:165698] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n18:165698] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n18:121308] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n18:121308] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n18:121308] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n15:164416] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n15:164416] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n15:164416] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n15:164416] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n15:164416] [ 9] [0x3ffffdc7fa60] +[a34n13:137749] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n13:137749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n13:137749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n13:137749] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n13:137749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n12:180115] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n12:180115] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n12:180115] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n12:180115] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n12:180115] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n12:180115] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n12:180115] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n12:180115] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n12:180115] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n05:135121] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n05:135121] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n05:135121] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n05:135121] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n05:135121] *** End of error message *** +[b20n07:132088] *** Process received signal *** +[b20n07:132088] Signal: Segmentation fault (11) +[b20n07:132088] Signal code: (3) +[b20n07:132088] Failing at address: 0xb170000000000a0 +[b20n07:132088] [ 0] [0x1000000504d8] +[b20n07:132088] [ 1] [0x3fffdac37360] +[b20n07:132088] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n07:132088] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n07:132088] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n04:138160] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n04:138160] *** End of error message *** +[a32n09:155758] *** Process received signal *** +[a32n09:155758] Signal: Segmentation fault (11) +[a32n09:155758] Signal code: (3) +[a32n09:155758] Failing at address: 0xb170000000000a0 +[a32n09:155758] [ 0] [0x1000000504d8] +[a32n09:155758] [ 1] [0x3fffd40e6ca0] +[a32n09:155758] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n09:155758] [ 3] [b03n16:173911] *** Process received signal *** +[b03n16:173911] Signal: Segmentation fault (11) +[b03n16:173911] Signal code: (3) +[b03n16:173911] Failing at address: 0xb170000000000a0 +[b03n16:173911] [ 0] [0x1000000504d8] +[b03n16:173911] [ 1] [0x3fffc44bc360] +[b03n16:173911] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n16:173911] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n16:173911] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n06:117527] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n06:117527] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n06:117527] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n06:167369] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n06:167369] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n06:167369] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n08:138176] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n08:138176] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n08:138176] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n08:138176] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n08:163162] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n08:163162] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n11:62423] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n11:62423] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n11:62423] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n11:62423] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n13:130061] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n13:130061] [10] [b02n01:167302] *** Process received signal *** +[b02n01:167302] Signal: Segmentation fault (11) +[b02n01:167302] Signal code: (3) +[b02n01:167302] Failing at address: 0x14170000000000a0 +[b02n01:167302] [ 0] [b10n18:162421] *** Process received signal *** +[b10n18:162421] Signal: Segmentation fault (11) +[b10n18:162421] Signal code: (3) +[b10n18:162421] Failing at address: 0xb170000000000a0 +[b10n18:162421] [ 0] [a35n02:149228] *** Process received signal *** +[a35n02:149228] Signal: Segmentation fault (11) +[a35n02:149228] Signal code: (3) +[a35n02:149228] Failing at address: 0xb170000000000a0 +[a35n02:149228] [ 0] [0x1000000504d8] +[a35n02:149228] [ 1] [0x3ffff03c9be0] +[a35n02:149228] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n02:149228] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n02:149228] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n09:119636] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n09:119636] *** End of error message *** +[b07n16:166500] *** Process received signal *** +[b07n16:166500] Signal: Segmentation fault (11) +[b07n16:166500] Signal code: (3) +[b07n16:166500] Failing at address: 0x14170000000000a0 +[b07n16:166500] [ 0] [0x1000000504d8] +[b07n16:166500] [ 1] [0x3fffee2fd8d0] +[b07n16:166500] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n16:166500] [ 3] [b17n16:127351] *** Process received signal *** +[b17n16:127351] Signal: Segmentation fault (11) +[b17n16:127351] Signal code: (3) +[b17n16:127351] Failing at address: 0xb170000000000a0 +[b17n16:127351] [ 0] [0x1000000504d8] +[b17n16:127351] [ 1] [0x3fffc4503c60] +[b17n16:127351] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n16:127351] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n16:127351] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n06:171505] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n06:171505] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n06:171505] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n06:171505] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n06:171505] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n01:122014] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n01:122014] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n01:122014] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n01:122014] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n01:122014] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n01:122014] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n01:122014] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n01:122014] [14] [a34n16:135793] *** Process received signal *** +[a34n16:135793] Signal: Segmentation fault (11) +[a34n16:135793] Signal code: (3) +[a34n16:135793] Failing at address: 0x14170000000000a0 +[a34n16:135793] [ 0] [0x1000000504d8] +[a34n16:135793] [ 1] [0x3fffcfcf51f0] +[a34n16:135793] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n16:135793] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n16:135793] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n06:138767] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n06:138767] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n06:138767] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n06:138767] [ 8] [a28n17:168161] *** Process received signal *** +[a28n17:168161] Signal: Segmentation fault (11) +[a28n17:168161] Signal code: (3) +[a28n17:168161] Failing at address: 0xb170000000000a0 +[a28n17:168161] [ 0] [0x1000000504d8] +[a28n17:168161] [ 1] [0x3fffe1a5e2b0] +[a28n17:168161] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n17:168161] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n05:173821] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n05:173821] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n05:173821] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n05:173821] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n05:173821] [10] [b01n10:173680] *** Process received signal *** +[b01n10:173680] Signal: Segmentation fault (11) +[b01n10:173680] Signal code: (3) +[b01n10:173680] Failing at address: 0xb170000000000a0 +[b01n10:173680] [ 0] [0x1000000504d8] +[b01n10:173680] [ 1] [0x3fffc6e961a0] +[b01n10:173680] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n10:173680] [ 3] [a32n16:147288] *** Process received signal *** +[a32n16:147288] Signal: Segmentation fault (11) +[a32n16:147288] Signal code: (3) +[a32n16:147288] Failing at address: 0x14170000000000a0 +[b17n17:125810] *** Process received signal *** +[b17n17:125810] Signal: Segmentation fault (11) +[b17n17:125810] Signal code: (3) +[b17n17:125810] Failing at address: 0x14170000000000a0 +[b17n17:125810] [ 0] [0x1000000504d8] +[b17n17:125810] [ 1] [0x3ffff723f310] +[b17n17:125810] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n17:125810] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n17:125810] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n04:130746] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n04:130746] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n04:130746] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n04:130746] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n18:03316] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n18:03316] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n18:03316] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n18:03316] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n18:03316] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n18:03316] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n18:03316] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n05:02910] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n05:02910] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n05:02910] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n05:02910] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n05:02910] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n05:120232] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n05:120232] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n05:120232] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n05:120232] [ 8] [b20n09:121729] *** Process received signal *** +[b20n09:121729] Signal: Segmentation fault (11) +[b20n09:121729] Signal code: (3) +[b20n09:121729] Failing at address: 0xb170000000000a0 +[b20n09:121729] [ 0] [0x1000000504d8] +[b20n09:121729] [ 1] [0x3fffcef58900] +[b20n09:121729] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n09:121729] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n09:121729] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n14:139847] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n14:139847] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n14:139847] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n14:139847] [14] [b07n12:160961] *** Process received signal *** +[b07n12:160961] Signal: Segmentation fault (11) +[b07n12:160961] Signal code: (3) +[b07n12:160961] Failing at address: 0xb170000000000a0 +[b07n12:160961] [ 0] [0x1000000504d8] +[b07n12:160961] [ 1] [0x3fffde0ff6b0] +[b07n12:160961] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n12:160961] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n06:177005] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n06:177005] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n06:177005] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n06:177005] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n04:13084] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n04:13084] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n04:13084] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n04:13084] [ 8] [b07n14:176068] *** Process received signal *** +[b07n14:176068] Signal: Segmentation fault (11) +[b07n14:176068] Signal code: (3) +[b07n14:176068] Failing at address: 0x14170000000000a0 +[b07n14:176068] [ 0] [0x1000000504d8] +[b07n14:176068] [ 1] [0x3ffff6776500] +[b07n14:176068] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n14:176068] [ 3] [b01n02:01432] *** Process received signal *** +[b01n02:01432] Signal: Segmentation fault (11) +[b01n02:01432] Signal code: (3) +[b01n02:01432] Failing at address: 0x14170000000000a0 +[b01n02:01432] [ 0] [0x1000000504d8] +[b01n02:01432] [ 1] [0x3fffdf3dcde0] +[b01n02:01432] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n02:01432] [ 3] [a30n01:163392] *** Process received signal *** +[a30n01:163392] Signal: Segmentation fault (11) +[a30n01:163392] Signal code: (3) +[a30n01:163392] Failing at address: 0x14170000000000a0 +[a30n01:163392] [ 0] [0x1000000504d8] +[a30n01:163392] [ 1] [0x3fffdb7c1870] +[a30n01:163392] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n01:163392] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n01:163392] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n01:138005] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n01:138005] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n01:138005] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n01:138005] [12] [a31n07:156488] *** Process received signal *** +[a31n07:156488] Signal: Segmentation fault (11) +[a31n07:156488] Signal code: (3) +[a31n07:156488] Failing at address: 0xb170000000000a0 +[a31n07:156488] [ 0] [0x1000000504d8] +[a31n07:156488] [ 1] [0x3ffff4c16490] +[a31n07:156488] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n07:156488] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n07:156488] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n07:156488] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n06:162917] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n06:162917] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n06:162917] [ 6] [0x1000000504d8] +[b08n16:171917] [ 1] [0x3fffe32251f0] +[b08n16:171917] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n16:171917] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n16:171917] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n02:125375] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n02:125375] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n02:125375] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n02:125375] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n09:173751] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n09:173751] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n08:174413] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n08:174413] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n08:174413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n08:174413] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n08:174413] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n08:174413] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n08:174413] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n08:174413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n08:174413] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n08:174413] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n08:174413] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n15:177858] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n15:177858] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n15:177858] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n15:177858] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n15:177858] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n15:177858] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n15:177858] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n05:131264] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n05:131264] [10] [b06n17:163820] *** Process received signal *** +[b06n17:163820] Signal: Segmentation fault (11) +[b06n17:163820] Signal code: (3) +[b06n17:163820] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n05:139911] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n05:139911] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n05:139911] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n14:120640] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n14:120640] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n14:120640] [10] [a32n08:165286] *** Process received signal *** +[a32n08:165286] Signal: Segmentation fault (11) +[a32n08:165286] Signal code: (3) +[a32n08:165286] Failing at address: 0x14170000000000a0 +[a32n08:165286] [ 0] [b10n17:164561] *** Process received signal *** +[b10n17:164561] Signal: Segmentation fault (11) +[b10n17:164561] Signal code: (3) +[b10n17:164561] Failing at address: 0xb170000000000a0 +[b10n17:164561] [ 0] [0x1000000504d8] +[b10n17:164561] [ 1] [0x3fffc6fb49e0] +[b10n17:164561] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n17:164561] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n17:164561] [ 4] [a36n14:152041] *** Process received signal *** +[a36n14:152041] Signal: Segmentation fault (11) +[a36n14:152041] Signal code: (3) +[a36n14:152041] Failing at address: 0x14170000000000a0 +[a36n14:152041] [ 0] [0x1000000504d8] +[a36n14:152041] [ 1] [0x3ffffdd0ec90] +[a36n14:152041] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n14:152041] [ 3] [b10n08:129890] *** Process received signal *** +[b10n08:129890] Signal: Segmentation fault (11) +[b10n08:129890] Signal code: (3) +[b10n08:129890] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n07:138203] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n07:138203] [ 6] [a27n11:23063] *** Process received signal *** +[a27n11:23063] Signal: Segmentation fault (11) +[a27n11:23063] Signal code: (3) +[a27n11:23063] Failing at address: 0xb170000000000a0 +[a27n11:23063] [ 0] [b16n06:132108] *** Process received signal *** +[b16n06:132108] Signal: Segmentation fault (11) +[b16n06:132108] Signal code: (3) +[b16n06:132108] Failing at address: 0xb170000000000a0 +[b16n06:132108] [ 0] [0x1000000504d8] +[b16n06:132108] [ 1] [0x3ffff9bb9050] +[b16n06:132108] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n06:132108] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n18:126905] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n18:126905] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n18:126905] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n18:126905] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n18:126905] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n08:54625] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n14:133637] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n14:133637] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n14:133637] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n14:133637] [13] [b19n09:123417] *** Process received signal *** +[b19n09:123417] Signal: Segmentation fault (11) +[b19n09:123417] Signal code: (3) +[b19n09:123417] Failing at address: 0xb170000000000a0 +[b19n09:123417] [ 0] [0x1000000504d8] +[b19n09:123417] [ 1] [0x3fffcfc36160] +[b19n09:123417] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n09:123417] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n18:176811] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n18:176811] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n06:164015] [10] [0x3fffc75ce090] +[b21n08:116176] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n08:116176] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n08:116176] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n13:164689] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n13:164689] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n13:164689] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n13:164689] [10] [b12n03:139013] *** Process received signal *** +[b12n03:139013] Signal: Segmentation fault (11) +[b12n03:139013] Signal code: (3) +[b12n03:139013] Failing at address: 0x14170000000000a0 +[b12n03:139013] [ 0] [0x1000000504d8] +[b12n03:139013] [ 1] [0x3fffcbfdabc0] +[b12n03:139013] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n03:139013] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n03:139013] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n14:169525] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n14:169525] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n14:169525] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n14:169525] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n13:01303] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n13:01303] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n13:01303] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n13:01303] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n14:150135] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n14:150135] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n14:150135] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n14:150135] [ 8] [a35n14:136663] *** Process received signal *** +[a35n14:136663] Signal: Segmentation fault (11) +[a35n14:136663] Signal code: (3) +[a35n14:136663] Failing at address: 0x5170000000000a0 +[a35n14:136663] [ 0] [0x1000000504d8] +[a35n14:136663] [ 1] [0x3ffffb8b1c40] +[a35n14:136663] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n14:136663] [ 3] [a26n10:67859] *** Process received signal *** +[a26n10:67859] Signal: Segmentation fault (11) +[a26n10:67859] Signal code: (3) +[a26n10:67859] Failing at address: 0xb170000000000a0 +[a26n10:67859] [ 0] [0x1000000504d8] +[a26n10:67859] [ 1] [0x3fffd450f5e0] +[a26n10:67859] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n10:67859] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n06:135852] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n06:135852] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n06:135852] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n06:135852] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n04:172234] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n04:172234] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n04:172234] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n05:164946] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n05:164946] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n05:164946] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n05:164946] [13] [b22n17:119584] *** Process received signal *** +[b22n17:119584] Signal: Segmentation fault (11) +[b22n17:119584] Signal code: (3) +[b22n17:119584] Failing at address: 0xb170000000000a0 +[b22n17:119584] [ 0] [0x1000000504d8] +[b22n17:119584] [ 1] [0x3ffffb5b2460] +[b22n17:119584] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n17:119584] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n17:119584] [ 4] [b06n01:165167] *** Process received signal *** +[b06n01:165167] Signal: Segmentation fault (11) +[b06n01:165167] Signal code: (3) +[b06n01:165167] Failing at address: 0x5170000000000a0 +[b06n01:165167] [ 0] [0x1000000504d8] +[b06n01:165167] [ 1] [0x3fffe664e200] +[b06n01:165167] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n01:165167] [ 3] [b21n17:118576] *** Process received signal *** +[b21n17:118576] Signal: Segmentation fault (11) +[b21n17:118576] Signal code: (3) +[b21n17:118576] Failing at address: 0x14170000000000a0 +[b21n17:118576] [ 0] [0x1000000504d8] +[b21n17:118576] [ 1] [0x3fffe0633c10] +[b21n17:118576] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n17:118576] [ 3] [b14n17:119469] *** Process received signal *** +[b14n17:119469] Signal: Segmentation fault (11) +[b14n17:119469] Signal code: (3) +[b14n17:119469] Failing at address: 0x14170000000000a0 +[b14n17:119469] [ 0] [0x1000000504d8] +[b14n17:119469] [ 1] [0x3fffec65d8d0] +[b14n17:119469] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n17:119469] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n17:119469] [ 4] [b20n16:124666] *** Process received signal *** +[b20n16:124666] Signal: Segmentation fault (11) +[b20n16:124666] Signal code: (3) +[b20n16:124666] Failing at address: 0xb170000000000a0 +[b20n16:124666] [ 0] [0x1000000504d8] +[b20n16:124666] [ 1] [0x3fffc555f000] +[b20n16:124666] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n16:124666] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n16:124666] [ 4] [a32n13:163193] *** Process received signal *** +[a32n13:163193] Signal: Segmentation fault (11) +[a32n13:163193] Signal code: (3) +[a32n13:163193] Failing at address: 0xb170000000000a0 +[a32n13:163193] [ 0] [0x1000000504d8] +[a32n13:163193] [ 1] [0x3fffd4732280] +[a32n13:163193] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n13:163193] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n13:163193] [ 4] [a29n13:165391] *** Process received signal *** +[a29n13:165391] Signal: Segmentation fault (11) +[a29n13:165391] Signal code: (3) +[a29n13:165391] Failing at address: 0x14170000000000a0 +[a29n13:165391] [ 0] [0x1000000504d8] +[a29n13:165391] [ 1] [0x3fffcd73ed40] +[a29n13:165391] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n13:165391] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n13:165391] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n05:158577] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n05:158577] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n05:173083] *** End of error message *** +[b18n06:133954] *** Process received signal *** +[b18n06:133954] Signal: Segmentation fault (11) +[b18n06:133954] Signal code: (3) +[b18n06:133954] Failing at address: 0xb170000000000a0 +[b18n06:133954] [ 0] [0x1000000504d8] +[b18n06:133954] [ 1] [0x3ffff3d00930] +[b18n06:133954] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n06:133954] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n06:133954] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n15:119253] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n03:154297] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n03:154297] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n03:154297] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n03:154297] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n18:138476] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n18:138476] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n05:04522] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n05:04522] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n05:04522] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n05:04522] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n07:02842] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n07:02842] [10] [0x1000000504d8] +[a29n17:169736] [ 1] [0x3ffff8078480] +[a29n17:169736] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n17:169736] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n17:169736] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n17:169736] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n12:134495] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n12:134495] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n04:52708] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n04:52708] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n15:139751] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n15:139751] *** End of error message *** +[b23n08:116907] *** Process received signal *** +[b23n08:116907] Signal: Segmentation fault (11) +[b23n08:116907] Signal code: (3) +[b23n08:116907] Failing at address: 0x14170000000000a0 +[b23n08:116907] [ 0] [0x1000000504d8] +[b23n08:116907] [ 1] [0x3fffece84a30] +[b23n08:116907] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n08:116907] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n08:116907] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n04:123056] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n04:123056] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n04:123056] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n04:123056] [13] [b07n02:174059] *** Process received signal *** +[b07n02:174059] Signal: Segmentation fault (11) +[b07n02:174059] Signal code: (3) +[b07n02:174059] Failing at address: 0x14170000000000a0 +[b07n02:174059] [ 0] [0x1000000504d8] +[b07n02:174059] [ 1] [0x3fffdae61d80] +[b07n02:174059] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n02:174059] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n02:174059] [ 4] [b21n07:117363] *** Process received signal *** +[b21n07:117363] Signal: Segmentation fault (11) +[b21n07:117363] Signal code: (3) +[b21n07:117363] Failing at address: 0x14170000000000a0 +[b21n07:117363] [ 0] [0x1000000504d8] +[b21n07:117363] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n12:141261] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n12:141261] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n12:141261] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n12:141261] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n18:123414] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n18:123414] *** End of error message *** +[a32n06:154621] *** Process received signal *** +[a32n06:154621] Signal: Segmentation fault (11) +[a32n06:154621] Signal code: (3) +[a32n06:154621] Failing at address: 0xb170000000000a0 +[a32n06:154621] [ 0] [0x1000000504d8] +[a32n06:154621] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n05:123935] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n05:123935] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n05:123935] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n05:123935] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n13:122141] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n13:122141] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n13:122141] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n13:122141] [ 8] [b11n16:154186] *** Process received signal *** +[b11n16:154186] Signal: Segmentation fault (11) +[b11n16:154186] Signal code: (3) +[b11n16:154186] Failing at address: 0xb170000000000a0 +[b11n16:154186] [ 0] [0x1000000504d8] +[b11n16:154186] [ 1] [0x3fffdb2cb910] +[b11n16:154186] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n16:154186] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n16:154186] [ 4] [a34n02:139632] *** Process received signal *** +[a34n02:139632] Signal: Segmentation fault (11) +[a34n02:139632] Signal code: (3) +[a34n02:139632] Failing at address: 0x14170000000000a0 +[a34n02:139632] [ 0] [0x1000000504d8] +[a34n02:139632] [ 1] [0x3ffff6e49af0] +[a34n02:139632] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n02:139632] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n02:139632] [ 4] [a28n12:177498] *** Process received signal *** +[a28n12:177498] Signal: Segmentation fault (11) +[a28n12:177498] Signal code: (3) +[a28n12:177498] Failing at address: 0x14170000000000a0 +[a28n12:177498] [ 0] [0x1000000504d8] +[a28n12:177498] [ 1] [0x3fffde1b04a0] +[a28n12:177498] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n12:177498] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n03:130347] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n03:130347] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n03:130347] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n03:130347] [ 8] [b15n15:122083] *** Process received signal *** +[b15n15:122083] Signal: Segmentation fault (11) +[b15n15:122083] Signal code: (3) +[b15n15:122083] Failing at address: 0x5170000000000a0 +[b15n15:122083] [ 0] [0x1000000504d8] +[b15n15:122083] [ 1] [0x3ffff626bd90] +[b15n15:122083] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n15:122083] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n18:69153] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n18:69153] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n18:69153] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n18:69153] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n02:116889] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n02:116889] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n02:116889] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n02:116889] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n02:116889] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n02:116889] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n02:116889] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n03:172651] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n03:172651] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n03:172651] *** End of error message *** +[b04n03:09643] *** Process received signal *** +[b04n03:09643] Signal: Segmentation fault (11) +[b04n03:09643] Signal code: (3) +[b04n03:09643] Failing at address: 0xb170000000000a0 +[b04n03:09643] [ 0] [0x1000000504d8] +[b04n03:09643] [ 1] [0x3fffc6aa1f70] +[b04n03:09643] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n03:09643] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n03:09643] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n10:124503] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n10:124503] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n10:124503] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n10:124503] [13] [b13n12:140125] *** Process received signal *** +[b13n12:140125] Signal: Segmentation fault (11) +[b13n12:140125] Signal code: (3) +[b13n12:140125] Failing at address: 0xb170000000000a0 +[b13n12:140125] [ 0] [0x1000000504d8] +[b13n12:140125] [ 1] [a32n17:138614] *** Process received signal *** +[a32n17:138614] Signal: Segmentation fault (11) +[a32n17:138614] Signal code: (3) +[a32n17:138614] Failing at address: 0x5170000000000a0 +[a32n17:138614] [ 0] [0x1000000504d8] +[a32n17:138614] [ 1] [0x3ffff15e2e70] +[a32n17:138614] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n17:138614] [ 3] [b20n06:130545] *** Process received signal *** +[b20n06:130545] Signal: Segmentation fault (11) +[b20n06:130545] Signal code: (3) +[b20n06:130545] Failing at address: 0x14170000000000a0 +[b20n06:130545] [ 0] [0x1000000504d8] +[b20n06:130545] [ 1] [0x3fffd0a2a9d0] +[b20n06:130545] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n06:130545] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n06:130545] [ 4] [a36n15:151250] *** Process received signal *** +[a36n15:151250] Signal: Segmentation fault (11) +[a36n15:151250] Signal code: (3) +[a36n15:151250] Failing at address: 0x5170000000000a0 +[a36n15:151250] [ 0] [0x1000000504d8] +[a36n15:151250] [ 1] [0x3ffffbe30110] +[a36n15:151250] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n15:151250] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n15:151250] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n15:151250] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n17:162362] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n17:162362] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n17:162362] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n17:162362] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n14:116412] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n14:116412] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n14:116412] [ 6] [b10n04:169414] *** Process received signal *** +[b10n04:169414] Signal: Segmentation fault (11) +[b10n04:169414] Signal code: (3) +[b10n04:169414] Failing at address: 0xb170000000000a0 +[b10n04:169414] [ 0] [0x1000000504d8] +[b10n04:169414] [ 1] [0x3fffe3365640] +[b10n04:169414] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n04:169414] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n04:169414] [ 4] [a28n15:07538] *** Process received signal *** +[a28n15:07538] Signal: Segmentation fault (11) +[a28n15:07538] Signal code: (3) +[a28n15:07538] Failing at address: 0x14170000000000a0 +[b15n02:122557] *** Process received signal *** +[b15n02:122557] Signal: Segmentation fault (11) +[b15n02:122557] Signal code: (3) +[b15n02:122557] Failing at address: 0xb170000000000a0 +[b15n02:122557] [ 0] [0x1000000504d8] +[b15n02:122557] [ 1] [0x3fffda9ddc30] +[b15n02:122557] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n02:122557] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n02:163323] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n02:163323] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n02:163323] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n02:163323] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n02:163323] [ 9] [b12n04:138157] *** Process received signal *** +[b12n04:138157] Signal: Segmentation fault (11) +[b12n04:138157] Signal code: (3) +[b12n04:138157] Failing at address: 0x5170000000000a0 +[b12n04:138157] [ 0] [0x1000000504d8] +[b12n04:138157] [ 1] [0x3fffea942fc0] +[b12n04:138157] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n04:138157] [ 3] [b02n13:171111] *** Process received signal *** +[b02n13:171111] Signal: Segmentation fault (11) +[b02n13:171111] Signal code: (3) +[b02n13:171111] Failing at address: 0x14170000000000a0 +[b02n13:171111] [ 0] [0x1000000504d8] +[b02n13:171111] [ 1] [0x3fffc91a7a40] +[b02n13:171111] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n13:171111] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n13:171111] [ 4] [b21n09:120694] *** Process received signal *** +[b21n09:120694] Signal: Segmentation fault (11) +[b21n09:120694] Signal code: (3) +[b21n09:120694] Failing at address: 0xb170000000000a0 +[b21n09:120694] [ 0] [0x1000000504d8] +[b21n09:120694] [ 1] [0x3fffee0adac0] +[b21n09:120694] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n09:120694] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n09:120694] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n09:120694] [ 5] [b07n01:166962] *** Process received signal *** +[b07n01:166962] Signal: Segmentation fault (11) +[b07n01:166962] Signal code: (3) +[b07n01:166962] Failing at address: 0xb170000000000a0 +[b07n01:166962] [ 0] [0x1000000504d8] +[b07n01:166962] [ 1] [0x3fffe7ea0fa0] +[b07n01:166962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n01:166962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n01:166962] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n16:122496] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n16:122496] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n16:122496] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n16:122496] [13] [a32n10:162609] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n01:146657] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n01:146657] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n01:146657] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n01:146657] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n10:126195] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n10:126195] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n10:126195] [ 6] [a29n16:157890] *** Process received signal *** +[a29n16:157890] Signal: Segmentation fault (11) +[a29n16:157890] Signal code: (3) +[a29n16:157890] Failing at address: 0xb170000000000a0 +[a29n16:157890] [ 0] [0x1000000504d8] +[a29n16:157890] [ 1] [0x3fffe7203a20] +[a29n16:157890] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n16:157890] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n13:130061] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n13:130061] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n13:130061] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n11:62423] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n11:62423] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n11:62423] [11] [a35n13:138092] *** Process received signal *** +[a35n13:138092] Signal: Segmentation fault (11) +[a35n13:138092] Signal code: (3) +[a35n13:138092] Failing at address: 0x5170000000000a0 +[a35n13:138092] [ 0] [0x1000000504d8] +[a35n13:138092] [ 1] [0x3ffff32b44b0] +[a35n13:138092] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n13:138092] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n13:138092] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n08:156373] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n06:157792] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n06:157792] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n06:157792] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n06:157792] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n06:157792] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n06:157792] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n06:157792] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n06:157792] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n06:157792] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n06:157792] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n15:139059] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n15:139059] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n15:139059] [ 7] [b01n18:179698] *** Process received signal *** +[b01n18:179698] Signal: Segmentation fault (11) +[b01n18:179698] Signal code: (3) +[b01n18:179698] Failing at address: 0xb170000000000a0 +[b01n18:179698] [ 0] [0x1000000504d8] +[b01n18:179698] [ 1] [0x3fffe3b1e840] +[b01n18:179698] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n18:179698] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n07:125452] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n07:125452] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n07:125452] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n07:125452] [14] [b22n11:120240] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n17:177405] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n17:177405] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n17:177405] [10] [b21n12:120129] *** Process received signal *** +[b21n12:120129] Signal: Segmentation fault (11) +[b21n12:120129] Signal code: (3) +[b21n12:120129] Failing at address: 0xb170000000000a0 +[b21n12:120129] [ 0] [0x1000000504d8] +[b21n12:120129] [ 1] [0x3fffc9f634b0] +[b21n12:120129] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n12:120129] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n11:120582] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n11:120582] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n11:120582] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n11:120582] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n11:120582] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n03:119832] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n03:119832] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n03:119832] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n09:155758] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n09:155758] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n09:155758] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n10:142931] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n10:142931] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n10:142931] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n10:142931] [14] [b11n04:166209] *** Process received signal *** +[b11n04:166209] Signal: Segmentation fault (11) +[b11n04:166209] Signal code: (3) +[b11n04:166209] Failing at address: 0xb170000000000a0 +[b11n04:166209] [ 0] [0x1000000504d8] +[b11n04:166209] [ 1] [0x3fffe4dc49d0] +[b11n04:166209] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n04:166209] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n04:166209] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n09:79008] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n09:79008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n09:79008] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n09:79008] [ 7] [0x1000000504d8] +[b12n12:138736] [ 1] [0x3fffe86e47f0] +[b12n12:138736] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n12:138736] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n12:138736] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n12:138736] [ 5] [b01n17:179117] *** Process received signal *** +[b01n17:179117] Signal: Segmentation fault (11) +[b01n17:179117] Signal code: (3) +[b01n17:179117] Failing at address: 0xb170000000000a0 +[b01n17:179117] [ 0] [0x1000000504d8] +[b01n17:179117] [ 1] [0x3fffcaf64d60] +[b01n17:179117] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n17:179117] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n17:179117] [ 4] [b22n12:122968] *** Process received signal *** +[b22n12:122968] Signal: Segmentation fault (11) +[b22n12:122968] Signal code: (3) +[b22n12:122968] Failing at address: 0xb170000000000a0 +[b22n12:122968] [ 0] [b06n14:173897] *** Process received signal *** +[b06n14:173897] Signal: Segmentation fault (11) +[b06n14:173897] Signal code: (3) +[b06n14:173897] Failing at address: 0x14170000000000a0 +[b06n14:173897] [ 0] [0x1000000504d8] +[b06n14:173897] [ 1] [0x3fffe1df89e0] +[b06n14:173897] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n14:173897] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n14:173897] [ 4] [b02n12:180118] *** Process received signal *** +[b02n12:180118] Signal: Segmentation fault (11) +[b02n12:180118] Signal code: (3) +[b02n12:180118] Failing at address: 0x14170000000000a0 +[b02n12:180118] [ 0] [0x1000000504d8] +[b02n12:180118] [ 1] [0x3ffffd42c110] +[b02n12:180118] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n12:180118] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n12:180118] [ 4] [b19n10:120979] *** Process received signal *** +[b19n10:120979] Signal: Segmentation fault (11) +[b19n10:120979] Signal code: (3) +[b19n10:120979] Failing at address: 0x14170000000000a0 +[b19n10:120979] [ 0] [0x1000000504d8] +[b19n10:120979] [ 1] [0x3fffdda879e0] +[b19n10:120979] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n10:120979] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n10:120979] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n06:10227] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n06:10227] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n06:10227] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n06:10227] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n12:123560] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n12:123560] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n12:123560] [11] [b20n15:132616] *** Process received signal *** +[b20n15:132616] Signal: Segmentation fault (11) +[b20n15:132616] Signal code: (3) +[b20n15:132616] Failing at address: 0xb170000000000a0 +[b20n15:132616] [ 0] [0x1000000504d8] +[b20n15:132616] [ 1] [0x3fffc1e04110] +[b20n15:132616] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n15:132616] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n15:132616] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n18:169360] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n18:169360] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n18:169360] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n18:169360] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n18:169360] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n17:163179] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n17:163179] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n17:163179] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n17:163179] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n17:163179] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n09:164365] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n09:164365] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n09:164365] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n09:164365] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n09:164365] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n01:141342] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n01:141342] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n01:141342] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n01:141342] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n01:141342] [ 8] [b14n15:123042] *** Process received signal *** +[b14n15:123042] Signal: Segmentation fault (11) +[b14n15:123042] Signal code: (3) +[b14n15:123042] Failing at address: 0x14170000000000a0 +[b14n15:123042] [ 0] [0x1000000504d8] +[b14n15:123042] [ 1] [0x3fffc8d89980] +[b14n15:123042] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n15:123042] [ 3] [a28n06:179452] *** Process received signal *** +[a28n06:179452] Signal: Segmentation fault (11) +[a28n06:179452] Signal code: (3) +[a28n06:179452] Failing at address: 0x14170000000000a0 +[a28n06:179452] [ 0] [0x1000000504d8] +[a28n06:179452] [ 1] [0x3ffffb3ac410] +[a28n06:179452] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n06:179452] [ 3] [b15n17:126528] *** Process received signal *** +[b15n17:126528] Signal: Segmentation fault (11) +[b15n17:126528] Signal code: (3) +[b15n17:126528] Failing at address: 0xb170000000000a0 +[b15n17:126528] [ 0] [0x1000000504d8] +[b15n17:126528] [ 1] [0x3fffef24b560] +[b15n17:126528] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n17:126528] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n17:126528] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n05:173821] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n05:173821] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n05:173821] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n05:173821] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n05:173821] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n17:168161] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n17:168161] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n17:168161] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n17:168161] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n17:168161] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n16:173911] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n16:173911] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n16:173911] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n16:173911] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n16:173911] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n16:173911] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n16:173911] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n16:173911] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n16:173911] [13] [b23n07:118165] *** Process received signal *** +[b23n07:118165] Signal: Segmentation fault (11) +[b23n07:118165] Signal code: (3) +[b23n07:118165] Failing at address: 0xb170000000000a0 +[b23n07:118165] [ 0] [0x1000000504d8] +[b23n07:118165] [ 1] [0x3fffe92be710] +[b23n07:118165] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n07:118165] [ 3] [b21n10:130581] *** Process received signal *** +[b21n10:130581] Signal: Segmentation fault (11) +[b21n10:130581] Signal code: (3) +[b21n10:130581] Failing at address: 0xb170000000000a0 +[b21n10:130581] [ 0] [0x1000000504d8] +[b21n10:130581] [ 1] [0x3fffd01ff0f0] +[b21n10:130581] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n10:130581] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n07:132088] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n07:132088] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n07:132088] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n07:132088] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n04:133905] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n04:133905] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n04:133905] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n04:133905] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n18:03316] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n18:164798] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n18:164798] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n18:164798] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n18:164798] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n14:139847] *** End of error message *** +[b07n06:177008] *** Process received signal *** +[b07n06:177008] Signal: Segmentation fault (11) +[b07n06:177008] Signal code: (3) +[b07n06:177008] Failing at address: 0x14170000000000a0 +[b07n06:177008] [ 0] [0x1000000504d8] +[b07n06:177008] [ 1] [0x3fffda7c2a40] +[b07n06:177008] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n04:13084] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n04:13084] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n18:10722] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n18:10722] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n18:10722] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n18:10722] [13] [a35n10:151894] *** Process received signal *** +[a35n10:151894] Signal: Segmentation fault (11) +[a35n10:151894] Signal code: (3) +[a35n10:151894] Failing at address: 0xb170000000000a0 +[a35n10:151894] [ 0] [0x1000000504d8] +[a35n10:151894] [ 1] [0x3ffff2c94300] +[a35n10:151894] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n10:151894] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n10:151894] [ 4] [b04n13:06926] *** Process received signal *** +[b04n13:06926] Signal: Segmentation fault (11) +[b04n13:06926] Signal code: (3) +[b04n13:06926] Failing at address: 0x14170000000000a0 +[b04n13:06926] [ 0] [0x1000000504d8] +[b04n13:06926] [ 1] [0x3ffffc274f50] +[b04n13:06926] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n13:06926] [ 3] [b15n14:133967] *** Process received signal *** +[b15n14:133967] Signal: Segmentation fault (11) +[b15n14:133967] Signal code: (3) +[b15n14:133967] Failing at address: 0xb170000000000a0 +[b15n14:133967] [ 0] [0x1000000504d8] +[b15n14:133967] [ 1] [0x3fffdce45890] +[b15n14:133967] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n14:133967] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n14:133967] [ 4] [b08n10:11490] *** Process received signal *** +[b08n10:11490] Signal: Segmentation fault (11) +[b08n10:11490] Signal code: (3) +[b08n10:11490] Failing at address: 0x14170000000000a0 +[b08n10:11490] [ 0] [0x1000000504d8] +[b08n10:11490] [ 1] [0x3fffe5dbdbc0] +[b08n10:11490] [ 2] [0x1000000504d8] +[b02n01:167302] [ 1] [0x3ffff0687ac0] +[b02n01:167302] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n01:167302] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n01:167302] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n04:117353] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n04:117353] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n04:117353] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n04:117353] [10] [b06n10:176262] *** Process received signal *** +[b06n10:176262] Signal: Segmentation fault (11) +[b06n10:176262] Signal code: (3) +[b06n10:176262] Failing at address: 0x14170000000000a0 +[b06n10:176262] [ 0] [0x1000000504d8] +[b06n10:176262] [ 1] [0x3ffffbd67150] +[b06n10:176262] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n10:176262] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n07:124350] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n07:124350] [ 6] [b20n17:122357] *** Process received signal *** +[b20n17:122357] Signal: Segmentation fault (11) +[b20n17:122357] Signal code: (3) +[b20n17:122357] Failing at address: 0xb170000000000a0 +[b20n17:122357] [ 0] [0x1000000504d8] +[b20n17:122357] [ 1] [0x3fffd13b75b0] +[b20n17:122357] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n17:122357] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n17:122357] [ 4] [a32n14:152746] *** Process received signal *** +[a32n14:152746] Signal: Segmentation fault (11) +[a32n14:152746] Signal code: (3) +[a32n14:152746] Failing at address: 0xb170000000000a0 +[a32n14:152746] [ 0] [0x1000000504d8] +[a32n14:152746] [ 1] [0x3ffffefd2060] +[a32n14:152746] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n14:152746] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n14:152746] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n03:135166] [ 6] [b08n16:171921] *** Process received signal *** +[b08n16:171921] Signal: Segmentation fault (11) +[b08n16:171921] Signal code: (3) +[b08n16:171921] Failing at address: 0x14170000000000a0 +[b08n16:171921] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n15:177858] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n15:177858] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n15:177858] *** End of error message *** +[a29n10:164938] *** Process received signal *** +[a29n10:164938] Signal: Segmentation fault (11) +[a29n10:164938] Signal code: (3) +[a29n10:164938] Failing at address: 0xb170000000000a0 +[a29n10:164938] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n10:173680] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n10:173680] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n10:173680] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n10:173680] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n10:173680] [ 8] [b15n11:134747] *** Process received signal *** +[b15n11:134747] Signal: Segmentation fault (11) +[b15n11:134747] Signal code: (3) +[b15n11:134747] Failing at address: 0xb170000000000a0 +[b15n11:134747] [ 0] [0x1000000504d8] +[b15n11:134747] [ 1] [0x3fffd9e7e820] +[b15n11:134747] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n11:134747] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n11:134747] [ 4] [b22n15:120059] *** Process received signal *** +[b22n15:120059] Signal: Segmentation fault (11) +[b22n15:120059] Signal code: (3) +[b22n15:120059] Failing at address: 0x14170000000000a0 +[b22n15:120059] [ 0] [0x1000000504d8] +[b22n15:120059] [ 1] [0x3ffff54d1ac0] +[b22n15:120059] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n15:120059] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n15:120059] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n01:121747] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n01:121747] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n01:121747] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n01:121747] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n01:121747] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n01:121747] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n01:121747] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n01:121747] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n01:121747] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n01:121747] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n01:121747] [14] [b16n13:122189] *** Process received signal *** +[b16n13:122189] Signal: Segmentation fault (11) +[b16n13:122189] Signal code: (3) +[b16n13:122189] Failing at address: 0xb170000000000a0 +[b16n13:122189] [ 0] [0x1000000504d8] +[b16n13:122189] [ 1] [0x3ffff6f0e010] +[b16n13:122189] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n13:122189] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n13:122189] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n13:114396] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n13:114396] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n13:114396] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n13:114396] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n13:114396] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n06:167369] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n06:167369] *** End of error message *** +[a27n05:53374] *** Process received signal *** +[a27n05:53374] Signal: Segmentation fault (11) +[a27n05:53374] Signal code: (3) +[a27n05:53374] Failing at address: 0xb170000000000a0 +[a27n05:53374] [ 0] [0x1000000504d8] +[a27n05:53374] [ 1] [a28n08:175809] *** Process received signal *** +[a28n08:175809] Signal: Segmentation fault (11) +[a28n08:175809] Signal code: (3) +[a28n08:175809] Failing at address: 0x5170000000000a0 +[a28n08:175809] [ 0] [0x1000000504d8] +[a28n08:175809] [ 1] [0x3fffe3589fa0] +[a28n08:175809] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n08:175809] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n08:175809] [ 4] [b09n07:173045] *** Process received signal *** +[b09n07:173045] Signal: Segmentation fault (11) +[b09n07:173045] Signal code: (3) +[b09n07:173045] Failing at address: 0xb170000000000a0 +[b09n07:173045] [ 0] [0x1000000504d8] +[b09n07:173045] [ 1] [0x3fffe55caf00] +[b09n07:173045] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n07:173045] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n07:173045] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n01:171789] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n01:171789] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n01:171789] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n01:171789] [13] [b12n09:128001] *** Process received signal *** +[b12n09:128001] Signal: Segmentation fault (11) +[b12n09:128001] Signal code: (3) +[b12n09:128001] Failing at address: 0xb170000000000a0 +[b12n09:128001] [ 0] [0x1000000504d8] +[b12n09:128001] [ 1] [0x3fffce8ad740] +[b12n09:128001] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n09:128001] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n09:128001] [ 4] [b01n09:173754] *** Process received signal *** +[b01n09:173754] Signal: Segmentation fault (11) +[b01n09:173754] Signal code: (3) +[b01n09:173754] Failing at address: 0x14170000000000a0 +[b01n09:173754] [ 0] [b18n03:136669] *** Process received signal *** +[b18n03:136669] Signal: Segmentation fault (11) +[b18n03:136669] Signal code: (3) +[b18n03:136669] Failing at address: 0xb170000000000a0 +[b18n03:136669] [ 0] [0x1000000504d8] +[b18n03:136669] [ 1] [0x3fffdd2200e0] +[b18n03:136669] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n03:136669] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n03:136669] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n06:163679] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n06:163679] *** End of error message *** +[b19n03:123616] *** End of error message *** +[b02n11:02687] *** Process received signal *** +[b02n11:02687] Signal: Segmentation fault (11) +[b02n11:02687] Signal code: (3) +[b02n11:02687] Failing at address: 0x14170000000000a0 +[b02n11:02687] [ 0] [0x1000000504d8] +[b02n11:02687] [ 1] [0x3ffff58fe960] +[b02n11:02687] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n11:02687] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n11:02687] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n03:125244] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n03:125244] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n03:125244] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n12:160961] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n12:160961] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n12:160961] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n12:160961] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n12:160961] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n14:122445] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n14:122445] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n14:122445] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n14:122445] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n01:164965] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n01:164965] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n01:164965] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n15:148168] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n15:148168] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n15:148168] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n15:148168] *** End of error message *** +[0x1000000504d8] +[a27n11:23063] [ 1] [0x3fffd200a6b0] +[a27n11:23063] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n11:23063] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n11:23063] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n11:23063] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n14:133637] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n14:133637] *** End of error message *** +[a27n08:54628] *** Process received signal *** +[a27n08:54628] Signal: Segmentation fault (11) +[a27n08:54628] Signal code: (3) +[a27n08:54628] Failing at address: 0x14170000000000a0 +[a27n08:54628] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n06:132108] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n06:132108] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n06:132108] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n06:132108] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n06:132108] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n06:132108] [ 9] [b04n18:176812] *** Process received signal *** +[b04n18:176812] Signal: Segmentation fault (11) +[b04n18:176812] Signal code: (3) +[b04n18:176812] Failing at address: 0x5170000000000a0 +[b04n18:176812] [ 0] [0x1000000504d8] +[b04n18:176812] [ 1] [0x3fffcce6d530] +[b04n18:176812] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n18:176812] [ 3] [b09n09:170659] *** Process received signal *** +[b09n09:170659] Signal: Segmentation fault (11) +[b09n09:170659] Signal code: (3) +[b09n09:170659] Failing at address: 0x14170000000000a0 +[b09n09:170659] [ 0] [0x1000000504d8] +[b09n09:170659] [ 1] [0x3ffff0009390] +[b09n09:170659] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n09:170659] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n06:164015] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n06:164015] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n06:164015] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n13:164689] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n13:164689] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n13:164689] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n14:169525] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n14:169525] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n14:169525] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n13:01303] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n13:01303] *** End of error message *** +[b01n08:172432] *** Process received signal *** +[b01n08:172432] Signal: Segmentation fault (11) +[b01n08:172432] Signal code: (3) +[b01n08:172432] Failing at address: 0x14170000000000a0 +[b01n08:172432] [ 0] [0x1000000504d8] +[b01n08:172432] [ 1] [0x3fffd4471100] +[b01n08:172432] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n08:172432] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n10:67859] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n10:67859] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n10:67859] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n06:135852] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n06:135852] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n06:135852] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n06:135852] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n13:123515] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n13:123515] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n13:123515] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n13:123515] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n18:165698] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n18:165698] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n18:121308] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n18:121308] [10] [b16n17:119979] *** Process received signal *** +[b16n17:119979] Signal: Segmentation fault (11) +[b16n17:119979] Signal code: (3) +[b16n17:119979] Failing at address: 0x14170000000000a0 +[b16n17:119979] [ 0] [0x1000000504d8] +[b16n17:119979] [ 1] [0x3fffcb794a40] +[b16n17:119979] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n17:119979] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n17:119584] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n17:119584] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n17:119584] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n17:119584] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n17:119584] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n17:118576] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n17:118576] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n17:118576] [ 6] [b07n11:170899] *** Process received signal *** +[b07n11:170899] Signal: Segmentation fault (11) +[b07n11:170899] Signal code: (3) +[b07n11:170899] Failing at address: 0x14170000000000a0 +[b07n11:170899] [ 0] [0x1000000504d8] +[b07n11:170899] [ 1] [0x3fffd54f91c0] +[b07n11:170899] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n11:170899] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n11:170899] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n08:116176] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n08:116176] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n13:165391] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n13:165391] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n13:165391] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n13:165391] [ 8] [a33n13:168160] *** Process received signal *** +[a33n13:168160] Signal: Segmentation fault (11) +[a33n13:168160] Signal code: (3) +[a33n13:168160] Failing at address: 0xb170000000000a0 +[a33n13:168160] [ 0] [0x1000000504d8] +[a33n13:168160] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n18:163108] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n18:163108] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n18:163108] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n18:163108] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n18:163108] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n07:156488] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n07:156488] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n07:156488] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n07:156488] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n07:156488] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n07:156488] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n07:156488] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n08:138176] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n08:138176] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n08:138176] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n08:138176] [11] [b16n01:132064] *** Process received signal *** +[b16n01:132064] Signal: Segmentation fault (11) +[b16n01:132064] Signal code: (3) +[b16n01:132064] Failing at address: 0x14170000000000a0 +[b16n01:132064] [ 0] [0x1000000504d8] +[b16n01:132064] [ 1] [0x3ffffe8e99e0] +[b16n01:132064] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n01:132064] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n16:166500] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n16:166500] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n16:166500] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n03:154297] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n03:154297] *** End of error message *** +[a32n18:138474] *** Process received signal *** +[a32n18:138474] Signal: Segmentation fault (11) +[a32n18:138474] Signal code: (3) +[a32n18:138474] Failing at address: 0xb170000000000a0 +[a32n18:138474] [ 0] [0x1000000504d8] +[a32n18:138474] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n05:04522] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n05:04522] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n05:04522] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n07:02842] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n07:02842] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n07:02842] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n07:02842] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n17:169736] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n17:169736] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n17:169736] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n17:169736] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n17:169736] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n04:123056] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n04:123056] *** End of error message *** +[b06n05:163965] *** Process received signal *** +[b06n05:163965] Signal: Segmentation fault (11) +[b06n05:163965] Signal code: (3) +[b06n05:163965] Failing at address: 0xb170000000000a0 +[b06n05:163965] [ 0] [0x1000000504d8] +[b06n05:163965] [ 1] [0x3fffe2e500b0] +[b06n05:163965] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n05:163965] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n05:163965] [ 4] [b18n13:123053] *** Process received signal *** +[b18n13:123053] Signal: Segmentation fault (11) +[b18n13:123053] Signal code: (3) +[b18n13:123053] Failing at address: 0xb170000000000a0 +[b18n13:123053] [ 0] [0x1000000504d8] +[b18n13:123053] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n15:139748] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n15:139748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n12:115638] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n12:115638] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n12:115638] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n12:115638] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n09:121729] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n09:121729] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n09:121729] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n03:139013] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n03:139013] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n03:139013] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n03:139013] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n03:139013] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n03:139013] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n03:139013] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n03:139013] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n03:139013] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n07:116116] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n07:116116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n07:116116] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n07:116116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n07:116116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n13:137749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n13:137749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n13:137749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n13:137749] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n01:122014] *** End of error message *** +[b17n15:137006] *** Process received signal *** +[b17n15:137006] Signal: Segmentation fault (11) +[b17n15:137006] Signal code: (3) +[b17n15:137006] Failing at address: 0xb170000000000a0 +[b17n15:137006] [ 0] [b18n15:120131] *** Process received signal *** +[b18n15:120131] Signal: Segmentation fault (11) +[b18n15:120131] Signal code: (3) +[b18n15:120131] Failing at address: 0xb170000000000a0 +[b18n15:120131] [ 0] [0x1000000504d8] +[b18n15:120131] [ 1] [b03n17:176829] *** Process received signal *** +[b03n17:176829] Signal: Segmentation fault (11) +[b03n17:176829] Signal code: (3) +[b03n17:176829] Failing at address: 0xb170000000000a0 +[b03n17:176829] [ 0] [0x1000000504d8] +[b03n17:176829] [ 1] [0x3fffcb2b6190] +[b03n17:176829] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n17:176829] [ 3] [b23n06:117530] *** Process received signal *** +[b23n06:117530] Signal: Segmentation fault (11) +[b23n06:117530] Signal code: (3) +[b23n06:117530] Failing at address: 0x14170000000000a0 +[b23n06:117530] [ 0] [0x1000000504d8] +[b23n06:117530] [ 1] [0x3ffffbebc6c0] +[b23n06:117530] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n06:117530] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n06:117530] [ 4] [b21n13:117509] *** Process received signal *** +[b21n13:117509] Signal: Segmentation fault (11) +[b21n13:117509] Signal code: (3) +[b21n13:117509] Failing at address: 0x14170000000000a0 +[b21n13:117509] [ 0] [b06n17:163820] [ 0] [0x1000000504d8] +[b06n17:163820] [ 1] [0x3fffec14ed70] +[b06n17:163820] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n17:163820] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n17:163820] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n07:117360] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n07:117360] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n07:117360] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n07:117360] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n07:117360] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n07:117360] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n07:117360] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n16:154186] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n16:154186] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n16:154186] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n16:154186] [ 8] [b22n04:44760] *** Process received signal *** +[b22n04:44760] Signal: Segmentation fault (11) +[b22n04:44760] Signal code: (3) +[b22n04:44760] Failing at address: 0x14170000000000a0 +[b22n04:44760] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n03:130347] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n03:130347] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n16:147285] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n16:147285] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n16:147285] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n16:147285] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n16:147285] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n16:147285] [a26n18:69152] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n15:122083] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n15:122083] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n15:122083] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n15:122083] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n15:122083] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n15:122083] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n15:122083] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n06:143013] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n06:143013] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n06:143013] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n06:143013] *** End of error message *** +[b10n11:160346] *** Process received signal *** +[b10n11:160346] Signal: Segmentation fault (11) +[b10n11:160346] Signal code: (3) +[b10n11:160346] Failing at address: 0x14170000000000a0 +[b10n11:160346] [ 0] [0x1000000504d8] +[b10n11:160346] [ 1] [0x3fffc3fdbd20] +[b10n11:160346] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n11:160346] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n11:160346] [ 4] [b11n07:156141] *** Process received signal *** +[b11n07:156141] Signal: Segmentation fault (11) +[b11n07:156141] Signal code: (3) +[b11n07:156141] Failing at address: 0xb170000000000a0 +[b11n07:156141] [ 0] [0x1000000504d8] +[b11n07:156141] [ 1] [0x3fffd1e4b8c0] +[b11n07:156141] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n07:156141] [ 3] [a34n12:145498] *** Process received signal *** +[a34n12:145498] Signal: Segmentation fault (11) +[a34n12:145498] Signal code: (3) +[a34n12:145498] Failing at address: 0x14170000000000a0 +[a34n12:145498] [ 0] [b20n11:123523] *** Process received signal *** +[b20n11:123523] Signal: Segmentation fault (11) +[b20n11:123523] Signal code: (3) +[b20n11:123523] Failing at address: 0x14170000000000a0 +[b20n11:123523] [ 0] [0x1000000504d8] +[b20n11:123523] [ 1] [0x3ffff6dc3770] +[b20n11:123523] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n11:123523] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n10:124503] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n10:124503] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n12:177498] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n12:177498] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n12:177498] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n12:177498] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n12:177498] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n12:177498] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n12:177498] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n12:177498] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n12:177498] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n12:177498] [13] [0x3ffff7463580] +[b13n12:140125] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n12:140125] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n12:140125] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n12:140125] [ 5] [b09n06:167094] *** Process received signal *** +[b09n06:167094] Signal: Segmentation fault (11) +[b09n06:167094] Signal code: (3) +[b09n06:167094] Failing at address: 0xb170000000000a0 +[b09n06:167094] [ 0] [0x1000000504d8] +[b09n06:167094] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n05:164946] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n05:164946] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n05:123935] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n05:123935] [10] [b15n09:121753] *** Process received signal *** +[b15n09:121753] Signal: Segmentation fault (11) +[b15n09:121753] Signal code: (3) +[b15n09:121753] Failing at address: 0x14170000000000a0 +[b15n09:121753] [ 0] [b05n15:176427] *** Process received signal *** +[b05n15:176427] Signal: Segmentation fault (11) +[b05n15:176427] Signal code: (3) +[b05n15:176427] Failing at address: 0x5170000000000a0 +[b05n15:176427] [ 0] [0x1000000504d8] +[b05n15:176427] [ 1] [0x3fffe4c8c0e0] +[b05n15:176427] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n15:176427] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n15:176427] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n04:130746] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n04:130746] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n04:130746] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n03:09643] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n03:09643] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n03:09643] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n03:09643] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n14:120640] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n14:120640] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n14:120640] [13] [b23n09:119633] *** Process received signal *** +[b23n09:119633] Signal: Segmentation fault (11) +[b23n09:119633] Signal code: (3) +[b23n09:119633] Failing at address: 0xb170000000000a0 +[b23n09:119633] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n02:164490] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n02:164490] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n02:164490] [ 6] [a35n07:138206] *** Process received signal *** +[a35n07:138206] Signal: Segmentation fault (11) +[a35n07:138206] Signal code: (3) +[a35n07:138206] Failing at address: 0x5170000000000a0 +[a35n07:138206] [ 0] [0x1000000504d8] +[a35n07:138206] [ 1] [0x3fffe6e01c60] +[a35n07:138206] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n05:158577] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n05:158577] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n05:158577] [14] [a34n08:163165] *** Process received signal *** +[a34n08:163165] Signal: Segmentation fault (11) +[a34n08:163165] Signal code: (3) +[a34n08:163165] Failing at address: 0x14170000000000a0 +[a34n08:163165] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n04:172234] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n04:172234] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n04:172234] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n04:172234] [14] [a27n01:48807] *** Process received signal *** +[a27n01:48807] Signal: Segmentation fault (11) +[a27n01:48807] Signal code: (3) +[a27n01:48807] Failing at address: 0x14170000000000a0 +[a27n01:48807] [ 0] [0x1000000504d8] +[a27n01:48807] [ 1] [0x3fffe641be70] +[a27n01:48807] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n01:48807] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n16:127351] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n16:127351] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n16:127351] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n16:127351] [ 8] [b16n11:124436] *** Process received signal *** +[b16n11:124436] Signal: Segmentation fault (11) +[b16n11:124436] Signal code: (3) +[b16n11:124436] Failing at address: 0xb170000000000a0 +[b16n11:124436] [ 0] [0x1000000504d8] +[b16n11:124436] [ 1] [0x3fffcf1f93f0] +[b16n11:124436] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n11:124436] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n11:124436] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n05:131264] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n05:131264] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n05:131264] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n05:120232] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n05:120232] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n05:120232] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n05:120232] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n05:120232] [13] [b07n14:176071] *** Process received signal *** +[b07n14:176071] Signal: Segmentation fault (11) +[b07n14:176071] Signal code: (3) +[b07n14:176071] Failing at address: 0x14170000000000a0 +[b07n14:176071] [ 0] [0x1000000504d8] +[b07n14:176071] [ 1] [0x3fffe99223d0] +[b07n14:176071] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n14:176071] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n02:149228] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n02:149228] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n17:119469] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n17:119469] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n17:119469] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n17:119469] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n17:119469] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n14:136663] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n14:136663] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n14:136663] [ 6] [a28n09:03929] *** Process received signal *** +[a28n09:03929] Signal: Segmentation fault (11) +[a28n09:03929] Signal code: (3) +[a28n09:03929] Failing at address: 0x14170000000000a0 +[a28n09:03929] [ 0] [0x1000000504d8] +[a28n09:03929] [ 1] [a28n15:07538] [ 0] [0x1000000504d8] +[a28n15:07538] [ 1] [0x3ffffdc9f1a0] +[a28n15:07538] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n15:07538] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n15:07538] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n15:07538] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n15:07538] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n09:123420] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n09:123420] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n09:123420] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n09:123420] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n02:174059] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n02:174059] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n02:174059] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n02:174059] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n04:138157] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n04:138157] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n17:125810] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n17:125810] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n17:125810] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n17:125810] [ 8] [0x1000000504d8] +[a32n08:165286] [ 1] [0x3fffec556a60] +[a32n08:165286] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n08:165286] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n08:165286] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n13:171111] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n13:171111] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n13:171111] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n13:171111] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n13:171111] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n15:164416] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n15:164416] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n15:164416] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n16:135793] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n16:135793] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n16:135793] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n16:135793] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n16:135793] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n16:135793] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n16:135793] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n16:135793] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n09:120694] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n09:120694] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n09:120694] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n09:120694] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n09:120694] [10] [b12n16:129238] *** Process received signal *** +[b12n16:129238] Signal: Segmentation fault (11) +[b12n16:129238] Signal code: (3) +[b12n16:129238] Failing at address: 0xb170000000000a0 +[b12n16:129238] [ 0] [0x1000000504d8] +[b12n16:129238] [ 1] [0x3fffde794de0] +[b12n16:129238] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n16:129238] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n16:129238] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n15:177057] *** Process received signal *** +[a29n15:177057] Signal: Segmentation fault (11) +[a29n15:177057] Signal code: (3) +[a29n15:177057] Failing at address: 0xb170000000000a0 +[a29n15:177057] [ 0] [0x1000000504d8] +[a29n15:177057] [ 1] [0x3ffff6ef5490] +[a29n15:177057] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n15:177057] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n17:162362] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n17:162362] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n17:162362] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n02:125375] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n14:116412] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n14:116412] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n14:116412] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n14:116412] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n11:156156] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n11:156156] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n11:156156] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n11:156156] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n11:156156] [ 9] [b02n15:169492] *** Process received signal *** +[b02n15:169492] Signal: Segmentation fault (11) +[b02n15:169492] Signal code: (3) +[b02n15:169492] Failing at address: 0xb170000000000a0 +[b02n15:169492] [ 0] [a32n07:172881] *** Process received signal *** +[a32n07:172881] Signal: Segmentation fault (11) +[a32n07:172881] Signal code: (3) +[a32n07:172881] Failing at address: 0x14170000000000a0 +[a32n07:172881] [ 0] [0x1000000504d8] +[b01n06:171502] [ 1] [0x3fffdeb83ae0] +[b01n06:171502] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n06:171502] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n06:171502] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n06:171502] [ 5] [b11n12:154346] *** Process received signal *** +[b11n12:154346] Signal: Segmentation fault (11) +[b11n12:154346] Signal code: (3) +[b11n12:154346] Failing at address: 0xb170000000000a0 +[b11n12:154346] [ 0] [0x1000000504d8] +[b11n12:154346] [ 1] [0x3ffff86f41f0] +[b11n12:154346] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n12:154346] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n12:154346] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n01:138005] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n01:138005] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n01:138005] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n08:174410] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n08:174410] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n08:174410] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n08:174410] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n08:174410] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n08:174410] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n08:174410] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n08:174410] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n08:174410] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n08:174410] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n08:174410] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n16:122493] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n16:122493] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n16:122493] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n16:122493] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n16:124666] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n16:124666] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n16:124666] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n16:124666] [ 8] [0x3ffff75be7c0] +[a32n06:154621] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n06:154621] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n06:154621] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n05:02910] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n05:02910] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n05:02910] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n05:02910] [13] [0x1000000504d8] +[b10n18:162421] [ 1] [0x3fffe5694de0] +[b10n18:162421] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n18:162421] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n18:162421] [ 4] [b14n02:139028] *** Process received signal *** +[b14n02:139028] Signal: Segmentation fault (11) +[b14n02:139028] Signal code: (3) +[b14n02:139028] Failing at address: 0x14170000000000a0 +[b14n02:139028] [ 0] [b02n02:166748] *** Process received signal *** +[b02n02:166748] Signal: Segmentation fault (11) +[b02n02:166748] Signal code: (3) +[b02n02:166748] Failing at address: 0xb170000000000a0 +[b02n02:166748] [ 0] [0x1000000504d8] +[b02n02:166748] [ 1] [0x3fffc1271f00] +[b02n02:166748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n02:166748] [ 3] [b23n11:117474] *** Process received signal *** +[b23n11:117474] Signal: Segmentation fault (11) +[b23n11:117474] Signal code: (3) +[b23n11:117474] Failing at address: 0xb170000000000a0 +[b23n11:117474] [ 0] [0x1000000504d8] +[b23n11:117474] [ 1] [0x3fffd19f1bb0] +[b23n11:117474] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n11:117474] [ 3] [a35n05:149467] *** Process received signal *** +[a35n05:149467] Signal: Segmentation fault (11) +[a35n05:149467] Signal code: (3) +[a35n05:149467] Failing at address: 0x14170000000000a0 +[a35n05:149467] [ 0] [0x1000000504d8] +[a35n05:149467] [ 1] [0x3fffd2cb81a0] +[a35n05:149467] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n05:149467] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n05:149467] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n01:166962] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n01:166962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n01:166962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n01:166962] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n01:166962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n01:166962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n01:166962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n01:166962] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n01:166962] [13] [b20n06:130542] *** Process received signal *** +[b20n06:130542] Signal: Segmentation fault (11) +[b20n06:130542] Signal code: (3) +[b20n06:130542] Failing at address: 0xb170000000000a0 +[b20n06:130542] [ 0] [0x1000000504d8] +[b20n06:130542] [ 1] [0x3fffd401bcc0] +[b20n06:130542] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n06:130542] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n16:157890] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n16:157890] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n16:157890] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n13:130061] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n13:130061] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n10:140417] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n10:140417] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n10:140417] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n10:140417] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n10:140417] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n10:140417] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n10:140417] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n10:140417] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n10:140417] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n11:62423] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n11:62423] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n11:62423] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n14:150135] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n14:150135] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n14:150135] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n06:157792] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n06:157792] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n02:01432] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n02:01432] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n02:01432] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n02:01432] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n02:01432] [ 8] [a28n01:179264] *** Process received signal *** +[a28n01:179264] Signal: Segmentation fault (11) +[a28n01:179264] Signal code: (3) +[a28n01:179264] Failing at address: 0x14170000000000a0 +[a28n01:179264] [ 0] [0x1000000504d8] +[a28n01:179264] [ 1] [0x3fffde927bd0] +[a28n01:179264] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n01:179264] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n01:179264] [ 4] [b09n15:165475] *** Process received signal *** +[b09n15:165475] Signal: Segmentation fault (11) +[b09n15:165475] Signal code: (3) +[b09n15:165475] Failing at address: 0xb170000000000a0 +[b09n15:165475] [ 0] [0x1000000504d8] +[b09n15:165475] [ 1] [0x3fffc09cbc70] +[b09n15:165475] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n15:165475] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n15:165475] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n06:133954] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n06:133954] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n06:133954] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n06:133954] [ 8] [b06n01:165164] *** Process received signal *** +[b06n01:165164] Signal: Segmentation fault (11) +[b06n01:165164] Signal code: (3) +[b06n01:165164] Failing at address: 0xb170000000000a0 +[b06n01:165164] [ 0] [0x1000000504d8] +[b06n01:165164] [ 1] [0x3fffca57f980] +[b06n01:165164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n01:165164] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n01:165164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n13:163196] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n13:163196] [ 5] [b11n18:163574] *** Process received signal *** +[b11n18:163574] Signal: Segmentation fault (11) +[b11n18:163574] Signal code: (3) +[b11n18:163574] Failing at address: 0x14170000000000a0 +[b11n18:163574] [ 0] [0x1000000504d8] +[b11n18:163574] [ 1] [0x3fffd257b220] +[b11n18:163574] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n18:163574] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n18:163574] [ 4] [a29n18:07426] *** Process received signal *** +[a29n18:07426] Signal: Segmentation fault (11) +[a29n18:07426] Signal code: (3) +[a29n18:07426] Failing at address: 0x14170000000000a0 +[a29n18:07426] [ 0] [0x1000000504d8] +[a29n18:07426] [ 1] [0x3fffd9b7c5a0] +[a29n18:07426] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n18:07426] [ 3] [b12n18:123868] *** Process received signal *** +[b12n18:123868] Signal: Segmentation fault (11) +[b12n18:123868] Signal code: (3) +[b12n18:123868] Failing at address: 0x14170000000000a0 +[b12n18:123868] [ 0] [0x1000000504d8] +[b12n18:123868] [ 1] [0x3fffc10423b0] +[b12n18:123868] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n18:123868] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n18:123868] [ 4] [b01n13:177639] *** Process received signal *** +[b01n13:177639] Signal: Segmentation fault (11) +[b01n13:177639] Signal code: (3) +[b01n13:177639] Failing at address: 0x5170000000000a0 +[b01n13:177639] [ 0] [0x1000000504d8] +[b01n13:177639] [ 1] [0x3fffcd1aeb70] +[b01n13:177639] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n13:177639] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n08:116904] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n08:116904] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n08:116904] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n08:116904] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n13:122141] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n13:122141] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n13:122141] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n05:139911] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n05:139911] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n05:139911] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n18:179698] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n18:179698] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n18:179698] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n18:179698] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n18:179698] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n14:152041] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n14:152041] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n14:152041] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n02:139632] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n02:139632] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n02:139632] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n02:139632] [ 8] [b22n11:120240] Signal: Segmentation fault (11) +[b22n11:120240] Signal code: (3) +[b22n11:120240] Failing at address: 0xb170000000000a0 +[b22n11:120240] [ 0] [0x1000000504d8] +[b22n11:120240] [ 1] [0x3fffe1cfb750] +[b22n11:120240] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n11:120240] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n11:120582] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n11:120582] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n11:120582] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n11:120582] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n11:120582] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n07:125452] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n06:162917] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n06:162917] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n06:162917] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n06:162917] [10] [b14n18:131586] *** Process received signal *** +[b14n18:131586] Signal: Segmentation fault (11) +[b14n18:131586] Signal code: (3) +[b14n18:131586] Failing at address: 0xb170000000000a0 +[b14n18:131586] [ 0] [0x1000000504d8] +[b14n18:131586] [ 1] [0x3fffcbdbd1f0] +[b14n18:131586] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n18:131586] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n18:131586] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n18:131586] [ 5] [a31n17:156183] *** Process received signal *** +[a31n17:156183] Signal: Segmentation fault (11) +[a31n17:156183] Signal code: (3) +[a31n17:156183] Failing at address: 0xb170000000000a0 +[a31n17:156183] [ 0] [0x1000000504d8] +[a31n17:156183] [ 1] [0x3fffe81ad4f0] +[a31n17:156183] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n17:156183] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n17:156183] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n17:156183] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n17:156183] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n17:156183] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n17:156183] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n17:156183] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n17:156183] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n17:156183] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n17:156183] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n17:156183] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n17:156183] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n13:138092] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n13:138092] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n13:138092] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n13:138092] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n13:138092] [ 9] [a33n15:148584] *** Process received signal *** +[a33n15:148584] Signal: Segmentation fault (11) +[a33n15:148584] Signal code: (3) +[a33n15:148584] Failing at address: 0x5170000000000a0 +[a33n15:148584] [ 0] [0x1000000504d8] +[a33n15:148584] [ 1] [0x3ffff728e6b0] +[a33n15:148584] [ 2] [a26n07:60174] *** Process received signal *** +[a26n07:60174] Signal: Segmentation fault (11) +[a26n07:60174] Signal code: (3) +[a26n07:60174] Failing at address: 0xb170000000000a0 +[a26n07:60174] [ 0] [0x1000000504d8] +[a26n07:60174] [ 1] [0x3fffc03f3a20] +[a26n07:60174] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n07:60174] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n07:60174] [ 4] [b18n17:124168] *** Process received signal *** +[b18n17:124168] Signal: Segmentation fault (11) +[b18n17:124168] Signal code: (3) +[b18n17:124168] Failing at address: 0x14170000000000a0 +[b18n17:124168] [ 0] [0x1000000504d8] +[b18n17:124168] [ 1] [0x3fffe1ff1e90] +[b18n17:124168] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n17:124168] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n17:124168] [ 4] [b08n17:171202] *** Process received signal *** +[b08n17:171202] Signal: Segmentation fault (11) +[b08n17:171202] Signal code: (3) +[b08n17:171202] Failing at address: 0xb170000000000a0 +[b08n17:171202] [ 0] [0x1000000504d8] +[b08n17:171202] [ 1] [0x3fffec0082f0] +[b08n17:171202] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n17:171202] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n17:171202] [ 4] [b22n03:120669] *** Process received signal *** +[b22n03:120669] Signal: Segmentation fault (11) +[b22n03:120669] Signal code: (3) +[b22n03:120669] Failing at address: 0xb170000000000a0 +[b22n03:120669] [ 0] [0x1000000504d8] +[b22n03:120669] [ 1] [0x3fffdb718930] +[b22n03:120669] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n03:120669] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n03:120669] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n04:166209] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n04:166209] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n04:166209] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n04:166209] [ 8] [0x1000000504d8] +[b22n12:122968] [ 1] [0x3fffe1bcc510] +[b22n12:122968] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n12:122968] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n12:122968] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n12:122968] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n17:179117] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n17:179117] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n17:179117] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n17:179117] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n17:179117] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n12:138736] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n12:138736] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n12:138736] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n09:79008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n09:79008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n09:79008] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n12:180115] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n12:180115] *** End of error message *** +[b15n10:122758] *** Process received signal *** +[b15n10:122758] Signal: Segmentation fault (11) +[b15n10:122758] Signal code: (3) +[b15n10:122758] Failing at address: 0xb170000000000a0 +[b15n10:122758] [ 0] [0x1000000504d8] +[b15n10:122758] [ 1] [0x3ffffb739f00] +[b15n10:122758] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n10:122758] [ 3] [b17n14:126123] *** Process received signal *** +[b17n14:126123] Signal: Segmentation fault (11) +[b17n14:126123] Signal code: (3) +[b17n14:126123] Failing at address: 0x14170000000000a0 +[b17n14:126123] [ 0] [0x1000000504d8] +[b17n14:126123] [ 1] [0x3fffd009ec40] +[b17n14:126123] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n14:126123] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n14:126123] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n02:163323] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n02:163323] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n02:163323] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n02:163323] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n15:132616] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n15:132616] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n15:132616] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n15:132616] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n12:123560] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n12:123560] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n12:123560] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n12:123560] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n06:10227] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n06:10227] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n06:10227] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n12:141261] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n12:141261] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n12:141261] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n12:141261] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n12:141261] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n16:173911] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n16:173911] *** End of error message *** +[b10n17:164564] *** Process received signal *** +[b10n17:164564] Signal: Segmentation fault (11) +[b10n17:164564] Signal code: (3) +[b10n17:164564] Failing at address: 0xb170000000000a0 +[b10n17:164564] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n01:141342] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n01:141342] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n01:141342] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n17:126528] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n17:126528] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n17:126528] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n17:126528] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n17:126528] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n17:168161] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n17:168161] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n17:168161] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n09:164365] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n09:164365] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n09:164365] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n09:164365] [13] [a31n18:169363] *** Process received signal *** +[a31n18:169363] Signal: Segmentation fault (11) +[a31n18:169363] Signal code: (3) +[a31n18:169363] Failing at address: 0x14170000000000a0 +[a31n18:169363] [ 0] [0x1000000504d8] +[a31n18:169363] [ 1] [0x3fffd0894a90] +[a31n18:169363] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n10:130584] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n10:130584] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n10:130584] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n07:132088] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n07:132088] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n07:132088] [11] [b12n14:139844] *** Process received signal *** +[b12n14:139844] Signal: Segmentation fault (11) +[b12n14:139844] Signal code: (3) +[b12n14:139844] Failing at address: 0xb170000000000a0 +[b12n14:139844] [ 0] [a36n17:150932] *** Process received signal *** +[a36n17:150932] Signal: Segmentation fault (11) +[a36n17:150932] Signal code: (3) +[a36n17:150932] Failing at address: 0x14170000000000a0 +[a36n17:150932] [ 0] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n06:177008] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n06:177008] [ 4] [b10n08:129890] [ 0] [0x1000000504d8] +[b10n08:129890] [ 1] [0x3fffc792a2a0] +[b10n08:129890] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n08:129890] [ 3] [a34n03:151708] *** Process received signal *** +[a34n03:151708] Signal: Segmentation fault (11) +[a34n03:151708] Signal code: (3) +[a34n03:151708] Failing at address: 0x14170000000000a0 +[a34n03:151708] [ 0] [0x1000000504d8] +[a34n03:151708] [ 1] [0x3fffe96311f0] +[a34n03:151708] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n03:151708] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n04:13084] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n04:13084] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n04:13084] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n04:13084] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n04:13084] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n18:10722] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n18:10722] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n14:133967] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n14:133967] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n14:133967] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n14:133967] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n13:06926] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n13:06926] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n13:06926] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n10:11493] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n10:11493] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n10:11493] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n16:171917] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n16:171917] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n16:171917] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n16:171917] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n01:167302] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n01:167302] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n17:122357] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n17:122357] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n17:122357] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n17:122357] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n17:122357] [ 9] [b15n11:134750] *** Process received signal *** +[b15n11:134750] Signal: Segmentation fault (11) +[b15n11:134750] Signal code: (3) +[b15n11:134750] Failing at address: 0xb170000000000a0 +[b15n11:134750] [ 0] [0x1000000504d8] +[b15n11:134750] [ 1] [0x3fffe186e710] +[b15n11:134750] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n11:134750] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n11:134750] [ 4] [0x1000000504d8] +[a29n10:164938] [ 1] [0x3ffff05a00a0] +[a29n10:164938] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n10:164938] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n10:164938] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n10:164938] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n10:173680] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n10:173680] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n10:173680] [11] [b04n15:177857] *** Process received signal *** +[b04n15:177857] Signal: Segmentation fault (11) +[b04n15:177857] Signal code: (3) +[b04n15:177857] Failing at address: 0xb170000000000a0 +[0x3fffc8ad38b0] +[a27n05:53374] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n05:53374] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n05:53374] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n05:53374] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n05:53374] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n13:122189] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n13:122189] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n13:122189] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n13:122189] [ 8] [b18n12:120403] *** Process received signal *** +[b18n12:120403] Signal: Segmentation fault (11) +[b18n12:120403] Signal code: (3) +[b18n12:120403] Failing at address: 0xb170000000000a0 +[b18n12:120403] [ 0] [0x1000000504d8] +[b18n12:120403] [ 1] [0x3ffffbd587b0] +[b18n12:120403] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n12:120403] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n12:120403] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n12:120403] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n12:120403] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n08:175809] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n08:175809] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n08:175809] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n08:175809] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n07:173045] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n07:173045] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n07:173045] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n07:173045] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n09:128001] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n09:128001] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n09:128001] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n12:160961] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n12:160961] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n12:160961] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n12:160961] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n11:02687] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n11:02687] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n11:02687] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n11:02687] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n11:02687] [ 9] [a35n15:148165] *** Process received signal *** +[a35n15:148165] Signal: Segmentation fault (11) +[a35n15:148165] Signal code: (3) +[a35n15:148165] Failing at address: 0xb170000000000a0 +[a35n15:148165] [ 0] [0x1000000504d8] +[a35n15:148165] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n11:23063] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n11:23063] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n11:23063] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n11:23063] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n11:23063] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n11:23063] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n11:23063] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n11:23063] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n09:170659] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n09:170659] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n09:170659] [ 6] [0x1000000504d8] +[a27n08:54628] [ 1] [0x3fffe9df0a60] +[a27n08:54628] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n08:54628] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n08:54628] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n06:132108] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n06:132108] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n06:132108] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n06:132108] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n06:132108] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n06:132108] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n06:164015] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n06:164015] *** End of error message *** +[b04n18:176810] [ 0] [0x1000000504d8] +[b04n18:176810] [ 1] [0x3ffff74e6f80] +[b04n18:176810] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n18:176810] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n18:176810] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n18:176810] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n13:164689] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n13:164689] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n14:169525] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n14:169525] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n14:169525] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n14:169525] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n10:67859] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n10:67859] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n10:67859] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n06:135852] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n07:118165] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n07:118165] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n07:118165] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n07:118165] [ 7] [b23n04:117356] *** Process received signal *** +[b23n04:117356] Signal: Segmentation fault (11) +[b23n04:117356] Signal code: (3) +[b23n04:117356] Failing at address: 0x14170000000000a0 +[b23n04:117356] [ 0] [0x1000000504d8] +[b23n04:117356] [ 1] [0x3fffd9a1dea0] +[b23n04:117356] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n04:117356] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n15:151250] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n15:151250] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n15:151250] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n15:151250] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n15:151250] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n03:136669] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n03:136669] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n03:136669] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n03:136669] [ 8] [b10n16:164554] *** Process received signal *** +[b10n16:164554] Signal: Segmentation fault (11) +[b10n16:164554] Signal code: (3) +[b10n16:164554] Failing at address: 0xb170000000000a0 +[b10n16:164554] [ 0] [0x1000000504d8] +[b10n16:164554] [ 1] [0x3ffff8128600] +[b10n16:164554] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n16:164554] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n16:164554] [ 4] [a35n12:145267] *** Process received signal *** +[a35n12:145267] Signal: Segmentation fault (11) +[a35n12:145267] Signal code: (3) +[a35n12:145267] Failing at address: 0x5170000000000a0 +[a35n12:145267] [ 0] [a32n14:152749] *** Process received signal *** +[a32n14:152749] Signal: Segmentation fault (11) +[a32n14:152749] Signal code: (3) +[a32n14:152749] Failing at address: 0x14170000000000a0 +[a32n14:152749] [ 0] [0x1000000504d8] +[a32n14:152749] [ 1] [0x3fffff6c63a0] +[a32n14:152749] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n14:152749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n14:152749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n17:119979] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n17:119979] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n17:119979] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n17:119979] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n17:119584] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n17:119584] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n17:119584] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n17:119584] [13] [b20n10:131276] *** Process received signal *** +[b20n10:131276] Signal: Segmentation fault (11) +[b20n10:131276] Signal code: (3) +[b20n10:131276] Failing at address: 0xb170000000000a0 +[b20n10:131276] [ 0] [0x1000000504d8] +[b20n10:131276] [ 1] [0x3ffff7786950] +[b20n10:131276] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n10:131276] [ 3] [b20n08:124727] *** Process received signal *** +[b20n08:124727] Signal: Segmentation fault (11) +[b20n08:124727] Signal code: (3) +[b20n08:124727] Failing at address: 0x14170000000000a0 +[b20n08:124727] [ 0] [0x1000000504d8] +[b20n08:124727] [ 1] [0x3fffcb364980] +[b20n08:124727] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n08:124727] [ 3] [b04n17:08309] *** Process received signal *** +[b04n17:08309] Signal: Segmentation fault (11) +[b04n17:08309] Signal code: (3) +[b04n17:08309] Failing at address: 0xb170000000000a0 +[b04n17:08309] [ 0] [0x1000000504d8] +[b04n17:08309] [ 1] [0x3fffec5f7db0] +[b04n17:08309] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n17:08309] [ 3] [a26n13:54243] *** Process received signal *** +[a26n13:54243] Signal: Segmentation fault (11) +[a26n13:54243] Signal code: (3) +[a26n13:54243] Failing at address: 0x14170000000000a0 +[a26n13:54243] [ 0] [0x1000000504d8] +[a26n13:54243] [ 1] [0x3fffee6c2000] +[a26n13:54243] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n13:54243] [ 3] [0x1000000504d8] +[b01n09:173754] [ 1] [0x3fffcd845dc0] +[b01n09:173754] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n09:173754] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n09:173754] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n17:118576] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n17:118576] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n17:118576] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n17:118576] [10] [b03n18:177224] *** Process received signal *** +[b03n18:177224] Signal: Segmentation fault (11) +[b03n18:177224] Signal code: (3) +[b03n18:177224] Failing at address: 0xb170000000000a0 +[b03n18:177224] [ 0] [0x1000000504d8] +[b03n18:177224] [ 1] [0x3fffd4b86880] +[b03n18:177224] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n18:177224] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n18:177224] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n17:177405] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n17:177405] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n17:177405] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n17:177405] [14] [b14n06:127274] *** Process received signal *** +[b14n06:127274] Signal: Segmentation fault (11) +[b14n06:127274] Signal code: (3) +[b14n06:127274] Failing at address: 0xb170000000000a0 +[b14n06:127274] [ 0] [0x1000000504d8] +[b14n06:127274] [ 1] [0x3fffc258e1f0] +[b14n06:127274] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n06:127274] [ 3] [b03n12:167676] *** Process received signal *** +[b03n12:167676] Signal: Segmentation fault (11) +[b03n12:167676] Signal code: (3) +[b03n12:167676] Failing at address: 0xb170000000000a0 +[b03n12:167676] [ 0] [0x1000000504d8] +[b03n12:167676] [ 1] [0x3fffee20b0b0] +[b03n12:167676] [ 2] [a29n11:162474] *** Process received signal *** +[a29n11:162474] Signal: Segmentation fault (11) +[a29n11:162474] Signal code: (3) +[a29n11:162474] Failing at address: 0x14170000000000a0 +[a29n11:162474] [ 0] [0x1000000504d8] +[a29n11:162474] [ 1] [0x3fffd08f6c60] +[a29n11:162474] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n11:162474] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n11:162474] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n01:146657] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n08:172432] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n08:172432] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n08:172432] [ 6] [b14n09:119550] *** Process received signal *** +[b14n09:119550] Signal: Segmentation fault (11) +[b14n09:119550] Signal code: (3) +[b14n09:119550] Failing at address: 0x14170000000000a0 +[b14n09:119550] [ 0] [0x1000000504d8] +[b14n09:119550] [ 1] [0x3fffcf9ce4e0] +[b14n09:119550] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n09:119550] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n09:119550] [ 4] [b14n08:127447] *** Process received signal *** +[b14n08:127447] Signal: Segmentation fault (11) +[b14n08:127447] Signal code: (3) +[b14n08:127447] Failing at address: 0x14170000000000a0 +[b14n08:127447] [ 0] [0x1000000504d8] +[b14n08:127447] [ 1] [0x3fffe8f6b0c0] +[b14n08:127447] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n08:127447] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n08:127447] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n15:120059] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n15:120059] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n15:120059] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n15:120059] [ 8] [b10n01:165128] *** Process received signal *** +[b10n01:165128] Signal: Segmentation fault (11) +[b10n01:165128] Signal code: (3) +[b10n01:165128] Failing at address: 0x14170000000000a0 +[b10n01:165128] [ 0] [0x1000000504d8] +[b10n01:165128] [ 1] [0x3fffd1a89440] +[b10n01:165128] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n01:165128] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n01:165128] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n13:165391] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n13:165391] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n13:165391] [11] [ 1] [0x3ffff58c4c10] +[a33n13:168160] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n13:168160] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n13:168160] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n13:168160] [ 5] [b11n17:164298] *** Process received signal *** +[b11n17:164298] Signal: Segmentation fault (11) +[b11n17:164298] Signal code: (3) +[b11n17:164298] Failing at address: 0xb170000000000a0 +[b11n17:164298] [ 0] [b18n14:128872] *** Process received signal *** +[b18n14:128872] Signal: Segmentation fault (11) +[b18n14:128872] Signal code: (3) +[b18n14:128872] Failing at address: 0x14170000000000a0 +[b18n14:128872] [ 0] [0x1000000504d8] +[b18n14:128872] [ 1] [0x3ffff750d170] +[b18n14:128872] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n14:128872] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n14:128872] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n13:123515] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n13:123515] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n13:123515] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n13:123515] [14] [a36n09:140257] *** Process received signal *** +[a36n09:140257] Signal: Segmentation fault (11) +[a36n09:140257] Signal code: (3) +[a36n09:140257] Failing at address: 0x5170000000000a0 +[a36n09:140257] [ 0] [0x1000000504d8] +[a36n09:140257] [ 1] [0x3fffe7197f70] +[a36n09:140257] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n09:140257] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n09:140257] [ 4] [a34n18:140385] *** Process received signal *** +[a34n18:140385] Signal: Segmentation fault (11) +[a34n18:140385] Signal code: (3) +[a34n18:140385] Failing at address: 0xb170000000000a0 +[a34n18:140385] [ 0] [0x1000000504d8] +[a34n18:140385] [ 1] [0x3fffc67cc450] +[a34n18:140385] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n18:140385] [ 3] [a31n11:153417] *** Process received signal *** +[a31n11:153417] Signal: Segmentation fault (11) +[a31n11:153417] Signal code: (3) +[a31n11:153417] Failing at address: 0x5170000000000a0 +[a31n11:153417] [ 0] [0x1000000504d8] +[a31n11:153417] [ 1] [0x3fffdb855380] +[a31n11:153417] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n11:153417] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n13:114399] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n13:114399] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n13:114399] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n13:114399] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n13:114399] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n11:170899] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n11:170899] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n11:170899] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n11:170899] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n11:170899] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n08:116176] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n08:116176] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n08:116176] [ 9] [b14n05:126780] *** Process received signal *** +[b14n05:126780] Signal: Segmentation fault (11) +[b14n05:126780] Signal code: (3) +[b14n05:126780] Failing at address: 0xb170000000000a0 +[b14n05:126780] [ 0] [0x1000000504d8] +[b14n05:126780] [ 1] [0x3fffd6f52d50] +[b14n05:126780] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n05:126780] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n10:151894] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n10:151894] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n10:151894] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n10:151894] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n10:151894] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n10:151894] [10] [a27n09:51335] *** Process received signal *** +[a27n09:51335] Signal: Segmentation fault (11) +[a27n09:51335] Signal code: (3) +[a27n09:51335] Failing at address: 0x14170000000000a0 +[a27n09:51335] [ 0] [0x1000000504d8] +[a27n09:51335] [ 1] [0x3fffe8073940] +[a27n09:51335] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n09:51335] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n01:132064] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n01:132064] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n01:132064] [ 6] [a35n08:138179] *** Process received signal *** +[a35n08:138179] Signal: Segmentation fault (11) +[a35n08:138179] Signal code: (3) +[a35n08:138179] Failing at address: 0x5170000000000a0 +[a35n08:138179] [ 0] [0x1000000504d8] +[a35n08:138179] [ 1] [0x3fffd958ef80] +[a35n08:138179] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n08:138179] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n07:156488] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n07:156488] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n07:156488] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n16:166500] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n16:166500] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n16:166500] [ 9] [0x3fffc4686650] +[a32n18:138474] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n18:138474] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n18:138474] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n05:04522] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n05:04522] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n07:02842] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n17:169736] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n17:169736] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n17:169736] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n17:169736] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n17:169736] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n10:176262] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n10:176262] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n10:176262] [ 6] [b02n17:03847] *** Process received signal *** +[b02n17:03847] Signal: Segmentation fault (11) +[b02n17:03847] Signal code: (3) +[b02n17:03847] Failing at address: 0xb170000000000a0 +[b02n17:03847] [ 0] [0x1000000504d8] +[b02n17:03847] [ 1] [0x3fffc177e440] +[b02n17:03847] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n17:03847] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n17:03847] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n17:03847] [ 5] [b19n03:123619] *** Process received signal *** +[b19n03:123619] Signal: Segmentation fault (11) +[b19n03:123619] Signal code: (3) +[b19n03:123619] Failing at address: 0x14170000000000a0 +[b19n03:123619] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n15:139748] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n15:139748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n15:139748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n15:139748] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n03:139013] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n03:139013] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n14:122445] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n14:122445] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n17:176829] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n17:176829] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n17:176829] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n09:121729] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n09:121729] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n09:121729] [10] [0x1000000504d8] +[b21n13:117509] [ 1] [0x3fffd00301b0] +[b21n13:117509] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n13:117509] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n13:117509] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n16:154186] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n16:154186] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n16:154186] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n16:154186] [12] [0x3fffda6108c0] +[b21n07:117363] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n07:117363] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n07:117363] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n07:117363] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n17:163820] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n10:142931] *** End of error message *** +[b11n10:161718] *** Process received signal *** +[b11n10:161718] Signal: Segmentation fault (11) +[b11n10:161718] Signal code: (3) +[b11n10:161718] Failing at address: 0xb170000000000a0 +[b11n10:161718] [ 0] [0x1000000504d8] +[b11n10:161718] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n04:169414] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n04:169414] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n04:169414] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n04:169414] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n04:169414] [ 9] [b18n08:131381] *** Process received signal *** +[b18n08:131381] Signal: Segmentation fault (11) +[b18n08:131381] Signal code: (3) +[b18n08:131381] Failing at address: 0x14170000000000a0 +[b18n08:131381] [ 0] [0x1000000504d8] +[b18n08:131381] [ 1] [0x3fffc323bd00] +[b18n08:131381] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n08:131381] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n08:131381] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n03:130347] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n03:130347] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n03:130347] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n03:130347] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n03:130347] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n18:121308] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n18:121308] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n18:121308] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n18:121308] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n03:125244] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n03:125244] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n03:125244] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n18:69153] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n18:69153] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n18:69153] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n18:69153] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n18:69153] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n15:122083] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n15:122083] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n15:122083] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n15:122083] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n15:122083] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n12:140125] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n12:140125] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n12:140125] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n12:177498] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n12:177498] *** End of error message *** +[0x3fffc2903c80] +[b09n06:167094] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n06:167094] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n06:167094] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n06:167094] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n06:167094] [ 6] [b22n05:123938] *** Process received signal *** +[b22n05:123938] Signal: Segmentation fault (11) +[b22n05:123938] Signal code: (3) +[b22n05:123938] Failing at address: 0x14170000000000a0 +[b22n05:123938] [ 0] [0x1000000504d8] +[b22n05:123938] [ 1] [0x3fffe90fa850] +[b22n05:123938] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n05:123938] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n07:156141] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n07:156141] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n07:156141] [ 6] [0x1000000504d8] +[b15n09:121753] [ 1] [0x3fffe6021460] +[b15n09:121753] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n09:121753] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n09:121753] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n03:135166] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n03:135166] [ 8] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n03:09643] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n03:09643] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n07:138203] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n07:138203] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n07:138203] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n07:138203] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n07:138203] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n05:158577] *** End of error message *** +[b13n06:133800] *** Process received signal *** +[b13n06:133800] Signal: Segmentation fault (11) +[b13n06:133800] Signal code: (3) +[b13n06:133800] Failing at address: 0xb170000000000a0 +[b13n06:133800] [ 0] [0x1000000504d8] +[b13n06:133800] [ 1] [0x3ffffd5466c0] +[b13n06:133800] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n06:133800] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n06:133800] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n06:133800] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n06:133800] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n06:133800] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n06:133800] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n06:133800] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n06:133800] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n06:133800] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n06:133800] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n06:133800] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n06:133800] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n05:139435] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n05:139435] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n05:139435] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n05:139435] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n05:139435] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n05:139435] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n05:139435] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n05:139435] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n05:139435] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n05:139435] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n01:48807] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n01:48807] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n01:48807] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n01:48807] [ 7] [b16n11:124433] *** Process received signal *** +[b16n11:124433] Signal: Segmentation fault (11) +[b16n11:124433] Signal code: (3) +[b16n11:124433] Failing at address: 0x5170000000000a0 +[b16n11:124433] [ 0] [0x1000000504d8] +[b16n11:124433] [ 1] [0x3fffe8a80cd0] +[b16n11:124433] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n11:124433] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n11:124433] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n14:136663] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n14:136663] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n14:136663] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n14:136663] [10] [0x1000000504d8] +[b17n15:137006] [ 1] [0x3fffee43e500] +[b17n15:137006] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n15:137006] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n15:137006] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n05:163965] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n05:163965] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n05:163965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n05:163965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n05:163965] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n14:176068] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n14:176068] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n14:176068] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n14:176068] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n14:176068] [ 8] [a28n09:03930] *** Process received signal *** +[a28n09:03930] Signal: Segmentation fault (11) +[a28n09:03930] Signal code: (3) +[a28n09:03930] Failing at address: 0x14170000000000a0 +[a28n09:03930] [ 0] [0x1000000504d8] +[a28n09:03930] [ 1] [0x3ffff8e25910] +[a28n09:03930] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n09:03930] [ 3] [0x3fffe6d9d810] +[b18n13:123053] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n13:123053] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n13:123053] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n13:123053] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n15:07538] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n15:07538] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n15:07538] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n15:07538] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n15:07538] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n15:07538] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n02:174059] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n02:174059] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n02:174059] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n02:174059] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n04:138157] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n04:138157] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n13:171111] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n13:171111] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n13:171111] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n13:171111] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n03:119832] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n03:119832] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n03:119832] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n03:119832] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n09:120697] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n09:120697] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n09:120697] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n09:120697] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n09:120697] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n09:120697] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n09:120697] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n09:120697] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n09:120697] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n09:120697] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n09:120697] *** End of error message *** +[a32n10:162609] Signal: Segmentation fault (11) +[a32n10:162609] Signal code: (3) +[a32n10:162609] Failing at address: 0xb170000000000a0 +[a32n10:162609] [ 0] [0x1000000504d8] +[a32n10:162609] [ 1] [0x3fffd6692200] +[a32n10:162609] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n10:162609] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n14:120640] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n14:120640] *** End of error message *** +[b12n16:129238] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n16:129238] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n16:129238] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n16:129238] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n16:129238] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n16:129238] [10] [a29n15:177056] *** Process received signal *** +[a29n15:177056] Signal: Segmentation fault (11) +[a29n15:177056] Signal code: (3) +[a29n15:177056] Failing at address: 0x14170000000000a0 +[a29n15:177056] [ 0] [0x1000000504d8] +[a29n15:177056] [ 1] [0x3fffd26c5320] +[a29n15:177056] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n15:177056] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n10:126195] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n10:126195] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n10:126195] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n10:126195] [10] [a31n10:155398] *** Process received signal *** +[a31n10:155398] Signal: Segmentation fault (11) +[a31n10:155398] Signal code: (3) +[a31n10:155398] Failing at address: 0xb170000000000a0 +[a31n10:155398] [ 0] [0x1000000504d8] +[a31n10:155398] [ 1] [0x3fffce21bbb0] +[a31n10:155398] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n10:155398] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n10:155398] [ 4] [0x1000000504d8] +[b02n15:169492] [ 1] [0x3fffcc5480d0] +[b02n15:169492] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n15:169492] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n15:169492] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n15:169492] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n14:116412] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n14:116412] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n14:116412] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n14:116412] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n14:116412] *** End of error message *** +[b13n18:128541] *** Process received signal *** +[b13n18:128541] Signal: Segmentation fault (11) +[b13n18:128541] Signal code: (3) +[b13n18:128541] Failing at address: 0xb170000000000a0 +[b13n18:128541] [ 0] [0x1000000504d8] +[b13n18:128541] [ 1] [0x3fffe453c4c0] +[b13n18:128541] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n18:128541] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n18:128541] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n18:128541] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n18:128541] [ 6] [b09n11:171966] *** Process received signal *** +[b09n11:171966] Signal: Segmentation fault (11) +[b09n11:171966] Signal code: (3) +[b09n11:171966] Failing at address: 0xb170000000000a0 +[b09n11:171966] [ 0] [0x1000000504d8] +[b09n11:171966] [ 1] [0x3fffdbd08f80] +[b09n11:171966] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n11:171966] [ 3] [a32n17:138617] *** Process received signal *** +[a32n17:138617] Signal: Segmentation fault (11) +[a32n17:138617] Signal code: (3) +[a32n17:138617] Failing at address: 0x14170000000000a0 +[a32n17:138617] [ 0] [0x1000000504d8] +[a32n17:138617] [ 1] [0x3ffff9236db0] +[a32n17:138617] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n17:138617] [ 3] [b07n17:162359] *** Process received signal *** +[b07n17:162359] Signal: Segmentation fault (11) +[b07n17:162359] Signal code: (3) +[b07n17:162359] Failing at address: 0xb170000000000a0 +[b07n17:162359] [ 0] [0x1000000504d8] +[b07n17:162359] [ 1] [0x3ffffb08e930] +[b18n15:120131] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n15:120131] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n15:120131] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n12:115638] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n12:115638] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n12:115638] *** End of error message *** +[a31n01:173780] *** Process received signal *** +[a31n01:173780] Signal: Segmentation fault (11) +[a31n01:173780] Signal code: (3) +[a31n01:173780] Failing at address: 0xb170000000000a0 +[a31n01:173780] [ 0] [0x1000000504d8] +[a31n01:173780] [ 1] [0x3fffdca71cb0] +[a31n01:173780] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n01:173780] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n01:164965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n01:164965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n01:164965] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n16:157890] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n16:157890] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n16:157890] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n16:157890] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n11:62423] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n15:176427] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n15:176427] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n15:176427] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n15:176427] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n15:176427] [ 9] [b15n12:132104] *** Process received signal *** +[b15n12:132104] Signal: Segmentation fault (11) +[b15n12:132104] Signal code: (3) +[b15n12:132104] Failing at address: 0xb170000000000a0 +[b15n12:132104] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n16:127351] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n16:127351] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n16:127351] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n18:07426] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n18:07426] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n18:07426] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n18:07426] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n18:07426] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n18:07426] [ 9] [b22n13:122031] *** Process received signal *** +[b22n13:122031] Signal: Segmentation fault (11) +[b22n13:122031] Signal code: (3) +[b22n13:122031] Failing at address: 0xb170000000000a0 +[b22n13:122031] [ 0] [0x1000000504d8] +[b22n13:122031] [ 1] [0x3ffff2a904c0] +[b22n13:122031] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n13:122031] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n13:122031] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n18:123868] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n18:123868] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n18:123868] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n18:123868] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n18:123868] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n18:123868] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n18:123868] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n18:123868] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n06:117530] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n06:117530] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n06:117530] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n06:117530] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n17:163179] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n17:163179] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n17:163179] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n17:163179] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n17:163179] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n01:179264] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n01:179264] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n01:179264] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n01:179264] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n15:165475] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n15:165475] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n15:165475] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n15:165475] [ 8] [0x1000000504d8] +[b23n09:119633] [ 1] [0x3fffd9576a80] +[b23n09:119633] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n09:119633] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n09:119633] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n09:155758] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n09:155758] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n09:155758] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n09:155758] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n18:163574] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n18:163574] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n18:163574] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n18:163574] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n11:120240] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n11:120240] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n18:164798] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n13:137749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n13:137749] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n13:137749] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n11:120582] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n11:120582] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n11:156156] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n11:156156] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n11:156156] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n11:156156] [13] [a28n13:176365] *** Process received signal *** +[a28n13:176365] Signal: Segmentation fault (11) +[a28n13:176365] Signal code: (3) +[a28n13:176365] Failing at address: 0xb170000000000a0 +[a28n13:176365] [ 0] [0x1000000504d8] +[a28n13:176365] [ 1] [0x3fffeb1a8720] +[a28n13:176365] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n13:176365] [ 3] [0x1000000504d8] +[b14n02:139028] [ 1] [0x3fffcd094920] +[b14n02:139028] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n02:139028] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n02:139028] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n02:139028] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n02:139028] [ 6] [b01n14:175713] *** Process received signal *** +[b01n14:175713] Signal: Segmentation fault (11) +[b01n14:175713] Signal code: (3) +[b01n14:175713] Failing at address: 0x14170000000000a0 +[b01n14:175713] [ 0] [0x1000000504d8] +[b01n14:175713] [ 1] [0x3fffe93d8080] +[b01n14:175713] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n14:175713] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n14:175713] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n17:125810] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n17:125810] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n07:168965] *** Process received signal *** +[b08n07:168965] Signal: Segmentation fault (11) +[b08n07:168965] Signal code: (3) +[b08n07:168965] Failing at address: 0x14170000000000a0 +[b08n07:168965] [ 0] [0x1000000504d8] +[b08n07:168965] [ 1] [0x3ffffcff4ea0] +[b08n07:168965] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n07:168965] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n07:168965] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n05:120232] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n05:120232] *** End of error message *** +[b15n04:133902] *** Process received signal *** +[b15n04:133902] Signal: Segmentation fault (11) +[b15n04:133902] Signal code: (3) +[b15n04:133902] Failing at address: 0x14170000000000a0 +[b15n04:133902] [ 0] [0x1000000504d8] +[b15n04:133902] [ 1] [0x3fffde203c00] +[b15n04:133902] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n04:133902] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n16:124666] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n16:124666] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n18:163108] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n18:163108] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n18:163108] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n18:163108] *** End of error message *** +[0x1000000504d8] +[a34n12:145498] [ 1] [0x3fffc1596000] +[a34n12:145498] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n12:145498] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n12:145498] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n12:145498] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n12:145498] [ 6] [a31n16:162627] *** Process received signal *** +[a31n16:162627] Signal: Segmentation fault (11) +[a31n16:162627] Signal code: (3) +[a31n16:162627] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n07:60174] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n07:60174] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n07:60174] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n07:60174] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n07:60174] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n07:60174] [10] [b05n11:176584] *** Process received signal *** +[b05n11:176584] Signal: Segmentation fault (11) +[b05n11:176584] Signal code: (3) +[b05n11:176584] Failing at address: 0xb170000000000a0 +[b05n11:176584] [ 0] [0x1000000504d8] +[b05n11:176584] [ 1] [0x3fffdab8b400] +[b05n11:176584] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n11:176584] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n14:173897] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n14:173897] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n14:173897] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n14:173897] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n04:166209] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n04:166209] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n04:166209] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n04:166209] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n12:122968] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n12:122968] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n12:122968] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n12:180118] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n12:180118] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n12:180118] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n12:180118] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n12:180118] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n12:180118] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n12:138736] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n12:138736] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n12:138736] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n12:138736] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n09:79008] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n09:79008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n09:79008] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n09:79008] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n09:79008] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n12:120129] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n12:120129] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n12:120129] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n12:120129] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n12:120129] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n12:120129] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n12:120129] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n12:120129] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n12:120129] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n12:120129] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n12:120129] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n12:120129] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n15:123039] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n15:123039] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n15:123039] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n15:123039] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n15:123039] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n15:123039] [ 9] [a32n11:152818] *** Process received signal *** +[a32n11:152818] Signal: Segmentation fault (11) +[a32n11:152818] Signal code: (3) +[a32n11:152818] Failing at address: 0xb170000000000a0 +[a32n11:152818] [ 0] [0x1000000504d8] +[a32n11:152818] [ 1] [0x3fffc1f1c580] +[a32n11:152818] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n11:152818] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n06:10227] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n06:10227] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n06:10227] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n06:10227] *** End of error message *** +[b11n12:154343] *** Process received signal *** +[b11n12:154343] Signal: Segmentation fault (11) +[b11n12:154343] Signal code: (3) +[b11n12:154343] Failing at address: 0xb170000000000a0 +[b11n12:154343] [ 0] [0x1000000504d8] +[b11n12:154343] [ 1] [0x3fffd13927f0] +[b11n12:154343] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n12:154343] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n12:154343] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n12:154343] [ 5] [a34n11:147714] *** Process received signal *** +[a34n11:147714] Signal: Segmentation fault (11) +[a34n11:147714] Signal code: (3) +[a34n11:147714] Failing at address: 0x14170000000000a0 +[a34n11:147714] [ 0] [0x1000000504d8] +[a34n11:147714] [ 1] [0x3fffec207780] +[a34n11:147714] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n11:147714] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n11:147714] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n01:141345] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n01:141345] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n01:141345] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n17:168161] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n17:168161] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n17:168161] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n17:168161] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n17:126528] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n17:126528] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n17:126528] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n17:126528] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n17:126528] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n09:164365] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n09:164365] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n17:171202] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n17:171202] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n17:171202] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n17:171202] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n17:171202] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n10:122758] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n10:122758] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n10:122758] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n10:130581] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n10:130581] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n10:130581] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n10:130581] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n10:130581] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n10:130581] [ 9] [b05n02:176052] *** Process received signal *** +[b05n02:176052] Signal: Segmentation fault (11) +[b05n02:176052] Signal code: (3) +[b05n02:176052] Failing at address: 0x5170000000000a0 +[b05n02:176052] [ 0] [0x1000000504d8] +[b05n02:176052] [ 1] [0x3fffc2ae4240] +[b05n02:176052] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n02:176052] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n02:176052] [ 4] [0x1000000504d8] +[b12n14:139844] [ 1] [0x3fffe4e705d0] +[b12n14:139844] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n14:139844] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n14:139844] [ 4] [b01n16:166691] *** Process received signal *** +[b01n16:166691] Signal: Segmentation fault (11) +[b01n16:166691] Signal code: (3) +[b01n16:166691] Failing at address: 0xb170000000000a0 +[b01n16:166691] [ 0] [0x1000000504d8] +[b01n16:166691] [ 1] [0x3ffff6d200c0] +[b01n16:166691] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n16:166691] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n05:131264] [14] [a32n16:147288] [ 0] [0x1000000504d8] +[a32n16:147288] [ 1] [0x3fffe45c1370] +[a32n16:147288] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n16:147288] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n16:147288] [ 4] [b06n07:174563] *** Process received signal *** +[b06n07:174563] Signal: Segmentation fault (11) +[b06n07:174563] Signal code: (3) +[b06n07:174563] Failing at address: 0x14170000000000a0 +[b06n07:174563] [ 0] [0x1000000504d8] +[b06n07:174563] [ 1] [0x3fffc7e13410] +[b06n07:174563] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n07:174563] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n07:174563] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n06:177008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n06:177008] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n06:177008] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n14:133967] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n14:133967] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n14:133967] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n14:133967] [12] [a26n14:51452] *** Process received signal *** +[a26n14:51452] Signal: Segmentation fault (11) +[a26n14:51452] Signal code: (3) +[a26n14:51452] Failing at address: 0x14170000000000a0 +[a26n14:51452] [ 0] [0x1000000504d8] +[a26n14:51452] [ 1] [0x3fffd0d7eda0] +[a26n14:51452] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n14:51452] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n16:171917] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n16:171917] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n16:171917] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n13:06926] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n13:06926] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n13:06926] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n13:06926] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n10:11490] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n10:11490] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n10:11490] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n10:11490] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n10:11490] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n10:11490] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n11:134747] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n11:134747] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n11:134747] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n11:134747] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n15:148584] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n15:148584] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n10:164938] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n10:164938] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n10:164938] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n10:164938] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n10:164938] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n10:164938] [11] [b04n15:177857] [ 0] [0x1000000504d8] +[b04n15:177857] [ 1] [0x3fffc485f0b0] +[b04n15:177857] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n15:177857] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n15:177857] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n15:177857] [ 5] [b15n08:118552] *** Process received signal *** +[b15n08:118552] Signal: Segmentation fault (11) +[b15n08:118552] Signal code: (3) +[b15n08:118552] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n13:122189] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n13:122189] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n13:122189] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n05:53374] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n05:53374] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n05:53374] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n05:53374] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n05:53374] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n05:53374] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n02:166748] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n02:166748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n02:166748] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n07:173045] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n07:173045] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n07:173045] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n06:133954] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n06:133954] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n06:133954] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n09:128001] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n09:128001] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n09:128001] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n12:160961] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n12:160961] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n12:160961] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n08:175809] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n08:175809] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n08:175809] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n09:123417] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n09:123417] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n09:123417] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n09:123417] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n09:123417] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n09:123417] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n11:02687] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n11:02687] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n11:02687] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n11:02687] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n09:170659] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n09:170659] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n09:170659] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n09:170659] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n08:54628] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n08:54628] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n08:54628] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n08:54628] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n02:163323] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n02:163323] *** End of error message *** +[b11n06:164016] *** Process received signal *** +[b11n06:164016] Signal: Segmentation fault (11) +[b11n06:164016] Signal code: (3) +[b11n06:164016] Failing at address: 0x14170000000000a0 +[b11n06:164016] [ 0] [b20n12:119807] *** Process received signal *** +[b20n12:119807] Signal: Segmentation fault (11) +[b20n12:119807] Signal code: (3) +[b20n12:119807] Failing at address: 0xb170000000000a0 +[b20n12:119807] [ 0] [0x1000000504d8] +[b20n12:119807] [ 1] [0x3fffd5ceac40] +[b20n12:119807] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n12:119807] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n10:67859] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n10:67859] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n10:67859] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n10:67859] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n18:176812] [ 4] [b09n13:164692] *** Process received signal *** +[b09n13:164692] Signal: Segmentation fault (11) +[b09n13:164692] Signal code: (3) +[b09n13:164692] Failing at address: 0xb170000000000a0 +[b09n13:164692] [ 0] [b14n17:119466] *** Process received signal *** +[b14n17:119466] Signal: Segmentation fault (11) +[b14n17:119466] Signal code: (3) +[b14n17:119466] Failing at address: 0xb170000000000a0 +[b14n17:119466] [ 0] [0x1000000504d8] +[b14n17:119466] [ 1] [0x3fffc8f51ea0] +[b14n17:119466] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n17:119466] [ 3] [0x1000000504d8] +[a34n08:163165] [ 1] [0x3fffe3f173b0] +[a34n08:163165] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n08:163165] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n08:163165] [ 4] [b16n17:119979] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n17:119979] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n17:119979] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n17:119979] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n17:08309] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n17:08309] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n17:08309] [ 6] [a27n15:32401] *** Process received signal *** +[a27n15:32401] Signal: Segmentation fault (11) +[a27n15:32401] Signal code: (3) +[a27n15:32401] Failing at address: 0x14170000000000a0 +[a27n15:32401] [ 0] [0x1000000504d8] +[a27n15:32401] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n18:179698] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n18:179698] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n18:179698] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n18:179698] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n17:118576] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n17:118576] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n17:118576] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n17:118576] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n13:54243] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n13:54243] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n13:54243] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n18:177224] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n18:177224] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n18:177224] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n18:177224] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n18:177224] [ 9] [b22n07:120699] *** Process received signal *** +[b22n07:120699] Signal: Segmentation fault (11) +[b22n07:120699] Signal code: (3) +[b22n07:120699] Failing at address: 0xb170000000000a0 +[b22n07:120699] [ 0] [0x1000000504d8] +[b22n07:120699] [ 1] [0x3fffd7e5f630] +[b22n07:120699] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n07:120699] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n11:162474] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n11:162474] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n11:162474] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n11:162474] [ 8] [b23n11:117477] *** Process received signal *** +[b23n11:117477] Signal: Segmentation fault (11) +[b23n11:117477] Signal code: (3) +[b23n11:117477] Failing at address: 0x14170000000000a0 +[b23n11:117477] [ 0] [0x1000000504d8] +[b23n11:117477] [ 1] [0x3fffd078cc60] +[b23n11:117477] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n11:117477] [ 3] [b07n13:165263] *** Process received signal *** +[b07n13:165263] Signal: Segmentation fault (11) +[b07n13:165263] Signal code: (3) +[b07n13:165263] Failing at address: 0x14170000000000a0 +[b07n13:165263] [ 0] [0x1000000504d8] +[b07n13:165263] [ 1] [0x3fffd7333550] +[b07n13:165263] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n13:165263] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n13:165263] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n06:130542] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n06:130542] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n06:130542] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n06:130542] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n06:130542] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n08:165286] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n08:165286] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n02:149228] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n02:149228] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n02:149228] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n02:149228] [10] [a33n02:167600] *** Process received signal *** +[a33n02:167600] Signal: Segmentation fault (11) +[a33n02:167600] Signal code: (3) +[a33n02:167600] Failing at address: 0x14170000000000a0 +[a33n02:167600] [ 0] [0x1000000504d8] +[a33n02:167600] [ 1] [0x3fffcd6517e0] +[a33n02:167600] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n02:167600] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n02:167600] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n07:116116] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n07:116116] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n07:116116] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n07:116116] [13] [b13n07:133374] *** Process received signal *** +[b13n07:133374] Signal: Segmentation fault (11) +[b13n07:133374] Signal code: (3) +[b13n07:133374] Failing at address: 0xb170000000000a0 +[b13n07:133374] [ 0] [0x1000000504d8] +[b13n07:133374] [ 1] [0x3fffc25ed4c0] +[b13n07:133374] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n07:133374] [ 3] [b09n16:173551] *** Process received signal *** +[b09n16:173551] Signal: Segmentation fault (11) +[b09n16:173551] Signal code: (3) +[b09n16:173551] Failing at address: 0xb170000000000a0 +[b09n16:173551] [ 0] [0x1000000504d8] +[b09n16:173551] [ 1] [0x3fffc17faa00] +[b09n16:173551] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n16:173551] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n16:173551] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n16:173551] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n16:173551] [ 6] [b01n13:177642] *** Process received signal *** +[b01n13:177642] Signal: Segmentation fault (11) +[b01n13:177642] Signal code: (3) +[b01n13:177642] Failing at address: 0x14170000000000a0 +[b01n13:177642] [ 0] [0x1000000504d8] +[b01n13:177642] [ 1] [0x3fffdbea73e0] +[b01n13:177642] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n13:177642] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n13:177642] [ 4] [0x1000000504d8] +[b22n04:44760] [ 1] [0x3fffef0f4e10] +[b22n04:44760] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n04:44760] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n04:44760] [ 4] [b21n14:119483] *** Process received signal *** +[b21n14:119483] Signal: Segmentation fault (11) +[b21n14:119483] Signal code: (3) +[b21n14:119483] Failing at address: 0x14170000000000a0 +[b21n14:119483] [ 0] [0x1000000504d8] +[b21n14:119483] [ 1] [0x3fffcd923430] +[b21n14:119483] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n14:119483] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n01:166962] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n01:166962] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n17:177405] *** End of error message *** +[0x1000000504d8] +[a32n07:172881] [ 1] [0x3fffde201870] +[a32n07:172881] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n07:172881] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n07:172881] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n13:165391] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n13:165391] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n13:165391] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n13:165391] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n13:168160] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n13:168160] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n13:168160] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n13:168160] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n13:168160] [10] [0x3fffe08f7730] +[a35n15:148165] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n15:148165] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n15:148165] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n15:148165] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n11:153417] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n11:153417] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n11:153417] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n11:170899] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n11:170899] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n11:170899] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n11:170899] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n08:116176] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n08:116176] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n08:116176] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n08:116176] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n17:179117] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n17:179117] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n17:179117] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n08:116907] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n08:116907] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n08:116907] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n08:116907] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n08:116907] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n08:116907] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n08:116907] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n08:116907] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n08:116907] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n08:116907] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n08:116907] *** End of error message *** +[b16n08:121435] *** Process received signal *** +[b16n08:121435] Signal: Segmentation fault (11) +[b16n08:121435] Signal code: (3) +[b16n08:121435] Failing at address: 0x14170000000000a0 +[b16n08:121435] [ 0] [0x1000000504d8] +[b16n08:121435] [ 1] [0x3fffdbf9ad70] +[b16n08:121435] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n08:121435] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n08:121435] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n16:166500] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n16:166500] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n16:166500] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n16:166500] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n16:122496] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n16:122496] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n18:138474] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n11:123523] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n11:123523] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n11:123523] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n11:123523] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n11:123523] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n11:123523] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n11:123523] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n11:123523] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n11:123523] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n11:123523] [13] [a28n05:04519] *** Process received signal *** +[a28n05:04519] Signal: Segmentation fault (11) +[a28n05:04519] Signal code: (3) +[a28n05:04519] Failing at address: 0x5170000000000a0 +[a28n05:04519] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n17:03847] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n17:03847] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n17:03847] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n17:03847] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n17:03847] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n17:03847] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n17:03847] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n17:03847] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n17:03847] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n17:03847] *** End of error message *** +[a36n07:153179] *** Process received signal *** +[a36n07:153179] Signal: Segmentation fault (11) +[a36n07:153179] Signal code: (3) +[a36n07:153179] Failing at address: 0x5170000000000a0 +[a36n07:153179] [ 0] [0x1000000504d8] +[a36n07:153179] [ 1] [0x3fffeef83430] +[a36n07:153179] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n07:153179] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n07:153179] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n06:154621] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n06:154621] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n11:160346] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n11:160346] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n11:160346] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n11:160346] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n15:139748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n15:139748] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n15:139748] [12] [a35n01:138008] *** Process received signal *** +[a35n01:138008] Signal: Segmentation fault (11) +[a35n01:138008] Signal code: (3) +[a35n01:138008] Failing at address: 0x14170000000000a0 +[a35n01:138008] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n05:149467] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n05:149467] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n05:149467] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n05:149467] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n17:176829] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n17:176829] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n17:176829] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n17:176829] [10] [b08n04:172231] *** Process received signal *** +[b08n04:172231] Signal: Segmentation fault (11) +[b08n04:172231] Signal code: (3) +[b08n04:172231] Failing at address: 0xb170000000000a0 +[b08n04:172231] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n13:117509] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n13:117509] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n16:154186] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n16:154186] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n16:154186] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n07:117360] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n07:117360] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n07:117360] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n07:117360] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n17:163820] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n14:126123] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n14:126123] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n14:126123] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n14:126123] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n15:120059] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n15:120059] [10] [b01n15:03735] *** Process received signal *** +[b01n15:03735] Signal: Segmentation fault (11) +[b01n15:03735] Signal code: (3) +[b01n15:03735] Failing at address: 0x14170000000000a0 +[b01n15:03735] [ 0] [0x1000000504d8] +[b01n15:03735] [ 1] [0x3fffcdfee480] +[b01n15:03735] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n15:03735] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n02:164490] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n02:164490] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n02:164490] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n02:164490] [10] [b02n16:01574] *** Process received signal *** +[b02n16:01574] Signal: Segmentation fault (11) +[b02n16:01574] Signal code: (3) +[b02n16:01574] Failing at address: 0xb170000000000a0 +[b02n16:01574] [ 0] [0x1000000504d8] +[b02n16:01574] [ 1] [0x3fffc97b63c0] +[b02n16:01574] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n16:01574] [ 3] [0x1000000504d8] +[a35n12:145267] [ 1] [0x3ffff40240c0] +[a35n12:145267] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n12:145267] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n12:145267] [ 4] [a27n04:52703] *** Process received signal *** +[a27n04:52703] Signal: Segmentation fault (11) +[a27n04:52703] Signal code: (3) +[a27n04:52703] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n09:51335] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n09:51335] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n09:51335] [ 6] [a29n01:153684] *** Process received signal *** +[a29n01:153684] Signal: Segmentation fault (11) +[a29n01:153684] Signal code: (3) +[a29n01:153684] Failing at address: 0x14170000000000a0 +[a29n01:153684] [ 0] [0x1000000504d8] +[a29n01:153684] [ 1] [0x3fffd62ef690] +[a29n01:153684] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n01:153684] [ 3] [b13n16:128440] *** Process received signal *** +[b13n16:128440] Signal: Segmentation fault (11) +[b13n16:128440] Signal code: (3) +[b13n16:128440] Failing at address: 0x14170000000000a0 +[b13n16:128440] [ 0] [0x1000000504d8] +[b13n16:128440] [ 1] [0x3fffd33d8a40] +[b13n16:128440] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n16:128440] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n01:167302] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n01:167302] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n01:167302] [ 9] [b16n18:134185] *** Process received signal *** +[b16n18:134185] Signal: Segmentation fault (11) +[b16n18:134185] Signal code: (3) +[b16n18:134185] Failing at address: 0xb170000000000a0 +[b16n18:134185] [ 0] [0x1000000504d8] +[b16n18:134185] [ 1] [0x3fffe39a7a10] +[b16n18:134185] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n18:134185] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n18:134185] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n01:132064] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n01:132064] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n01:132064] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n01:132064] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n13:122141] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n13:122141] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n13:122141] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n13:122141] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n15:132616] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n15:132616] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n15:132616] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n13:163193] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n13:163193] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n13:163193] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n13:163193] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n13:163193] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n13:163193] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n13:163193] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n13:163193] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n06:171505] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n06:171505] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n06:171505] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n06:171505] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n06:171505] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n06:171505] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n06:171505] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n14:150135] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n14:150135] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n14:150135] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n14:150135] *** End of error message *** +[b14n03:130350] *** Process received signal *** +[b14n03:130350] Signal: Segmentation fault (11) +[b14n03:130350] Signal code: (3) +[b14n03:130350] Failing at address: 0xb170000000000a0 +[b14n03:130350] [ 0] [a31n12:156919] *** Process received signal *** +[a31n12:156919] Signal: Segmentation fault (11) +[a31n12:156919] Signal code: (3) +[a31n12:156919] Failing at address: 0x14170000000000a0 +[a31n12:156919] [ 0] [0x1000000504d8] +[a31n12:156919] [ 1] [0x3fffc54d5b40] +[a31n12:156919] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n12:156919] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n12:156919] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n12:156919] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n12:156919] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n12:156919] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n12:156919] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n12:156919] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n12:156919] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n08:138176] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n08:138176] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n08:138176] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n08:138176] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n18:69152] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n06:167094] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n06:167094] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n06:167094] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n06:167094] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n06:167094] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n06:167094] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n06:167094] [13] [b05n01:178525] *** Process received signal *** +[b05n01:178525] Signal: Segmentation fault (11) +[b05n01:178525] Signal code: (3) +[b05n01:178525] Failing at address: 0x14170000000000a0 +[b05n01:178525] [ 0] [0x1000000504d8] +[b05n01:178525] [ 1] [0x3ffff3fa2720] +[b05n01:178525] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n01:178525] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n12:140125] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n12:140125] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n06:127274] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n06:127274] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n06:127274] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n06:127274] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n10:173680] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n10:173680] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n10:173680] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n03:136669] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n03:136669] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n05:123935] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n05:123935] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n05:123935] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n05:123935] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n05:123935] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n03:09643] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n03:09643] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n03:09643] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n03:09643] [14] [b18n17:124165] *** Process received signal *** +[b18n17:124165] Signal: Segmentation fault (11) +[b18n17:124165] Signal code: (3) +[b18n17:124165] Failing at address: 0xb170000000000a0 +[b18n17:124165] [ 0] [0x1000000504d8] +[b18n17:124165] [ 1] [0x3ffff17f77c0] +[b18n17:124165] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n17:124165] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n17:124165] [ 4] [a31n05:158574] *** Process received signal *** +[a31n05:158574] Signal: Segmentation fault (11) +[a31n05:158574] Signal code: (3) +[a31n05:158574] Failing at address: 0xb170000000000a0 +[a31n05:158574] [ 0] [0x1000000504d8] +[a31n05:158574] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n01:165167] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n01:165167] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n01:165167] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n01:165167] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n01:165167] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n01:165167] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n01:165167] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n01:165167] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n01:165167] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n18:131586] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n18:131586] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n18:131586] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n18:131586] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n18:131586] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n18:131586] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n18:131586] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n18:131586] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n17:156183] *** End of error message *** +[b20n07:132091] *** Process received signal *** +[b20n07:132091] Signal: Segmentation fault (11) +[b20n07:132091] Signal code: (3) +[b20n07:132091] Failing at address: 0x14170000000000a0 +[b20n07:132091] [ 0] [0x1000000504d8] +[b20n07:132091] [ 1] [0x3fffdedf7b40] +[b20n07:132091] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n07:132091] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n05:02910] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n05:02910] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n15:164416] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n15:164416] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n15:164416] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n13:138092] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n13:138092] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n13:138092] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n13:138092] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n13:138092] [14] [0x1000000504d8] +[b11n17:164298] [ 1] [0x3fffe64c9300] +[b11n17:164298] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n17:164298] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n17:164298] [ 4] [b13n02:125268] *** Process received signal *** +[b13n02:125268] Signal: Segmentation fault (11) +[b13n02:125268] Signal code: (3) +[b13n02:125268] Failing at address: 0x5170000000000a0 +[b13n02:125268] [ 0] [0x1000000504d8] +[b13n02:125268] [ 1] [0x3ffffeb87b50] +[b13n02:125268] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n02:125268] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n13:123515] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n09:173754] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n09:173754] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n12:120403] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n12:120403] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n12:120403] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n12:120403] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n12:120403] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n12:120403] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n12:120403] [13] [b03n14:177226] *** Process received signal *** +[b03n14:177226] Signal: Segmentation fault (11) +[b03n14:177226] Signal code: (3) +[b03n14:177226] Failing at address: 0x14170000000000a0 +[b03n14:177226] [ 0] [0x1000000504d8] +[b03n14:177226] [ 1] [0x3ffff93c52f0] +[b03n14:177226] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n14:177226] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n14:177226] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n17:122357] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n17:122357] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n17:122357] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n17:122357] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n17:122357] [14] [b22n16:124795] *** Process received signal *** +[b22n16:124795] Signal: Segmentation fault (11) +[b22n16:124795] Signal code: (3) +[b22n16:124795] Failing at address: 0xb170000000000a0 +[b22n16:124795] [ 0] [0x1000000504d8] +[b22n16:124795] [ 1] [0x3fffcf22d070] +[b22n16:124795] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n16:124795] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n16:124795] [ 4] [b14n11:130805] *** Process received signal *** +[b14n11:130805] Signal: Segmentation fault (11) +[b14n11:130805] Signal code: (3) +[b14n11:130805] Failing at address: 0xb170000000000a0 +[b14n11:130805] [ 0] [b11n10:161721] *** Process received signal *** +[b11n10:161721] Signal: Segmentation fault (11) +[b11n10:161721] Signal code: (3) +[b11n10:161721] Failing at address: 0xb170000000000a0 +[b11n10:161721] [ 0] [0x1000000504d8] +[b11n10:161721] [ 1] [0x3fffc903fa20] +[b11n10:161721] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n10:161721] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n10:161721] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n02:139632] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n02:139632] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n02:139632] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n11:124436] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n11:124436] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n11:124436] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n11:124436] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n11:124436] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n01:48807] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n01:48807] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n01:48807] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n05:163965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n05:163965] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n05:163965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n05:163965] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n14:176071] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n14:176071] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n14:176071] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n14:176071] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n14:176071] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n14:176071] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n14:176071] [10] [b05n13:10197] *** Process received signal *** +[b05n13:10197] Signal: Segmentation fault (11) +[b05n13:10197] Signal code: (3) +[b05n13:10197] Failing at address: 0xb170000000000a0 +[b05n13:10197] [ 0] [0x1000000504d8] +[b05n13:10197] [ 1] [0x3fffd93743a0] +[b05n13:10197] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n13:10197] [ 3] [0x3fffc55b9560] +[a28n09:03929] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n09:03929] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n09:03929] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n09:03929] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n09:03929] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n09:03929] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n09:03929] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n09:03929] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n09:03929] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n02:174059] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n02:174059] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n02:174059] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n15:07538] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n15:07538] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n15:07538] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n04:138157] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n04:138157] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n04:138157] [10] [b17n12:123235] *** Process received signal *** +[b17n12:123235] Signal: Segmentation fault (11) +[b17n12:123235] Signal code: (3) +[b17n12:123235] Failing at address: 0xb170000000000a0 +[b17n12:123235] [ 0] [0x1000000504d8] +[b17n12:123235] [ 1] [0x3ffffe815df0] +[b17n12:123235] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n12:123235] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n13:171111] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n13:171111] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n09:120694] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n09:120694] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n09:120694] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n09:120694] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n09:120694] *** End of error message *** +[b23n07:118168] *** Process received signal *** +[b23n07:118168] Signal: Segmentation fault (11) +[b23n07:118168] Signal code: (3) +[b23n07:118168] Failing at address: 0x14170000000000a0 +[b23n07:118168] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n16:129238] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n16:129238] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n16:129238] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n16:129238] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n16:129238] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n15:177057] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n15:177057] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n15:177057] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n15:177057] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n10:155398] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n10:155398] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n10:155398] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n10:155398] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n10:155398] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n07:156141] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n07:156141] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n07:156141] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n07:156141] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n15:169492] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n15:169492] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n15:169492] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n15:169492] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n15:169492] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n15:169492] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n15:169492] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n15:169492] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n05:139911] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n05:139911] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n05:139911] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n05:139911] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n18:169363] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n18:169363] [ 4] [a34n05:148379] *** Process received signal *** +[a34n05:148379] Signal: Segmentation fault (11) +[a34n05:148379] Signal code: (3) +[a34n05:148379] Failing at address: 0x14170000000000a0 +[a34n05:148379] [ 0] [0x1000000504d8] +[a34n05:148379] [ 1] [0x3fffe1e70a60] +[a34n05:148379] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n05:148379] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n05:148379] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n10:151894] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n10:151894] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n10:151894] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n10:140417] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n10:140417] *** End of error message *** +[b15n15:122086] *** Process received signal *** +[b15n15:122086] Signal: Segmentation fault (11) +[b15n15:122086] Signal code: (3) +[b15n15:122086] Failing at address: 0xb170000000000a0 +[b15n15:122086] [ 0] [0x1000000504d8] +[b15n15:122086] [ 1] [0x3fffc0f30590] +[b15n15:122086] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n15:122086] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n08:172432] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n08:172432] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n08:172432] [ 9] [b22n03:120672] *** Process received signal *** +[b22n03:120672] Signal: Segmentation fault (11) +[b22n03:120672] Signal code: (3) +[b22n03:120672] Failing at address: 0x14170000000000a0 +[b22n03:120672] [ 0] [0x1000000504d8] +[b22n03:120672] [ 1] [0x3ffffe9f61e0] +[b22n03:120672] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n03:120672] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n03:120672] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n04:117353] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n04:117353] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n04:117353] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n04:117353] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n10:131276] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n10:131276] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n10:131276] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n10:131276] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n10:131276] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n14:152041] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n14:152041] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n14:152041] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n14:152041] [10] [a35n09:137776] *** Process received signal *** +[a35n09:137776] Signal: Segmentation fault (11) +[a35n09:137776] Signal code: (3) +[a35n09:137776] Failing at address: 0x14170000000000a0 +[a35n09:137776] [ 0] [0x1000000504d8] +[a35n09:137776] [ 1] [0x3fffded6d4c0] +[a35n09:137776] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n09:137776] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n09:137776] [ 4] [a33n05:139438] *** Process received signal *** +[a33n05:139438] Signal: Segmentation fault (11) +[a33n05:139438] Signal code: (3) +[a33n05:139438] Failing at address: 0x14170000000000a0 +[a33n05:139438] [ 0] [b14n05:126783] *** Process received signal *** +[b14n05:126783] Signal: Segmentation fault (11) +[b14n05:126783] Signal code: (3) +[b14n05:126783] Failing at address: 0xb170000000000a0 +[b14n05:126783] [ 0] [0x1000000504d8] +[b14n05:126783] [ 1] [0x3ffff20cb4c0] +[b14n05:126783] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n05:126783] [ 3] [b13n05:135167] *** Process received signal *** +[b13n05:135167] Signal: Segmentation fault (11) +[b13n05:135167] Signal code: (3) +[b13n05:135167] Failing at address: 0xb170000000000a0 +[b13n05:135167] [ 0] [0x1000000504d8] +[b13n05:135167] [ 1] [0x3ffff03a92f0] +[b13n05:135167] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n05:135167] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n05:135167] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n08:131381] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n08:131381] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n08:131381] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n08:131381] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n08:131381] [ 9] [b22n17:119581] *** Process received signal *** +[b18n14:128869] *** Process received signal *** +[b18n14:128869] Signal: Segmentation fault (11) +[b18n14:128869] Signal code: (3) +[b18n14:128869] Failing at address: 0xb170000000000a0 +[b18n14:128869] [ 0] [0x1000000504d8] +[b18n14:128869] [ 1] [0x3fffd53ece80] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n13:114396] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n13:114396] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n13:114396] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n13:114396] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n01:163392] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n01:163392] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n01:163392] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n01:163392] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n01:163392] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n01:163392] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n01:163392] [11] [b20n08:124724] *** Process received signal *** +[b20n08:124724] Signal: Segmentation fault (11) +[b20n08:124724] Signal code: (3) +[b20n08:124724] Failing at address: 0xb170000000000a0 +[b20n08:124724] [ 0] [0x1000000504d8] +[b20n08:124724] [ 1] [0x3fffcb1b4740] +[b20n08:124724] [ 2] [0x3fffed37f850] +[b07n17:162359] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n17:162359] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n17:162359] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n17:162359] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n17:162359] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n17:162359] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n17:162359] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n17:162359] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n01:173780] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n01:173780] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n01:173780] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n01:173780] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n01:173780] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n01:173780] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n01:173780] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n01:173780] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n01:173780] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n01:173780] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n06:162917] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n06:162917] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n06:162917] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n16:157890] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n16:157890] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n16:157890] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n16:157890] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n16:157890] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n07:138203] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n07:138203] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n07:138203] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n07:138203] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n15:176427] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n15:176427] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n15:176427] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n15:176427] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n15:176427] [14] [b05n10:10927] *** Process received signal *** +[b05n10:10927] Signal: Segmentation fault (11) +[b05n10:10927] Signal code: (3) +[b05n10:10927] Failing at address: 0x14170000000000a0 +[b05n10:10927] [ 0] [0x1000000504d8] +[b05n10:10927] [ 1] [0x3fffefa16090] +[b05n10:10927] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n10:10927] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n15:137006] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n15:137006] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n15:137006] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n15:137006] [ 8] [0x1000000504d8] +[b19n03:123619] [ 1] [0x3fffc59b36d0] +[b19n03:123619] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n03:123619] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n03:123619] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n03:123619] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n07:124350] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n07:124350] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n07:124350] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n07:124350] [10] [0x1000000504d8] +[a36n17:150932] [ 1] [0x3fffdd3eca70] +[a36n17:150932] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n17:150932] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n14:152746] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n14:152746] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n14:152746] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n14:152746] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n14:152746] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n14:152746] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n17:164561] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n17:164561] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n17:164561] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n17:164561] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n17:164561] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n14:136663] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n14:136663] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n14:136663] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n14:136663] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n14:136663] *** End of error message *** +[a27n18:28218] *** Process received signal *** +[a27n18:28218] Signal: Segmentation fault (11) +[a27n18:28218] Signal code: (3) +[a27n18:28218] Failing at address: 0x14170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n06:133800] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n18:07426] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n18:07426] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n18:07426] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n18:07426] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n18:07426] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n18:07426] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n18:123868] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n18:123868] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n18:123868] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n01:179264] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n01:179264] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n01:179264] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n01:179264] [12] [b05n14:177514] *** Process received signal *** +[b05n14:177514] Signal: Segmentation fault (11) +[b05n14:177514] Signal code: (3) +[b05n14:177514] Failing at address: 0x14170000000000a0 +[b05n14:177514] [ 0] [0x1000000504d8] +[b05n14:177514] [ 1] [0x3fffdce11360] +[b05n14:177514] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n14:177514] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n16:127351] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n16:127351] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n16:127351] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n18:121308] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n18:163574] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n18:163574] [10] [b13n08:144856] *** Process received signal *** +[b13n08:144856] Signal: Segmentation fault (11) +[b13n08:144856] Signal code: (3) +[b13n08:144856] Failing at address: 0xb170000000000a0 +[b13n08:144856] [ 0] [0x1000000504d8] +[b13n08:144856] [ 1] [0x3fffe53b1170] +[b13n08:144856] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n08:144856] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n08:144856] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n11:156156] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n11:156156] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n13:176365] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n13:176365] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n13:176365] [ 6] [b21n11:120585] *** Process received signal *** +[b21n11:120585] Signal: Segmentation fault (11) +[b21n11:120585] Signal code: (3) +[b21n11:120585] Failing at address: 0x14170000000000a0 +[b21n11:120585] [ 0] [0x1000000504d8] +[b21n11:120585] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n09:119550] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n09:119550] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n09:119550] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n09:119550] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n09:119550] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n15:151250] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n15:151250] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n15:151250] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n08:129890] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n03:151708] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n03:151708] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n03:151708] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n03:151708] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n03:151708] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n03:151708] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n03:151708] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n15:165475] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n15:165475] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n15:165475] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n11:120240] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n11:120240] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n11:120240] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n13:123053] [ 6] [b23n12:115641] *** Process received signal *** +[b23n12:115641] Signal: Segmentation fault (11) +[b23n12:115641] Signal code: (3) +[b23n12:115641] Failing at address: 0x14170000000000a0 +[b23n12:115641] [ 0] [0x1000000504d8] +[b23n12:115641] [ 1] [a02n18:21770] *** Process received signal *** +[a02n18:21770] Signal: Segmentation fault (11) +[a02n18:21770] Signal code: (3) +[a02n18:21770] Failing at address: 0x14170000000000a0 +[a02n18:21770] [ 0] [0x1000000504d8] +[a02n18:21770] [ 1] [0x3ffff1fa2750] +[a02n18:21770] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n18:21770] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n09:121729] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n09:121729] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n09:121729] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n09:121729] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n09:140257] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n09:140257] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n09:140257] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n09:140257] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n16:164554] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n16:164554] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n16:164554] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n16:164554] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n16:164554] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n16:164554] [10] [a31n08:156370] *** Process received signal *** +[a31n08:156370] Signal: Segmentation fault (11) +[a31n08:156370] Signal code: (3) +[a31n08:156370] Failing at address: 0xb170000000000a0 +[a31n08:156370] [ 0] [a28n02:178478] *** Process received signal *** +[a28n02:178478] Signal: Segmentation fault (11) +[a28n02:178478] Signal code: (3) +[a28n02:178478] Failing at address: 0x14170000000000a0 +[a28n02:178478] [ 0] [0x1000000504d8] +[a28n02:178478] [ 1] [0x3fffc2678400] +[a28n02:178478] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n02:178478] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n02:178478] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n18:128541] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n18:128541] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n18:128541] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n18:128541] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n18:128541] [11] [b17n17:125810] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n17:125810] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n17:125810] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n17:125810] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n17:125810] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n15:120131] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n15:120131] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n06:117530] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n06:117530] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n06:117530] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n06:117530] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n06:117530] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n06:117530] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n06:117530] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n08:127447] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n08:127447] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n08:127447] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n08:127447] [ 8] [b12n02:143969] *** Process received signal *** +[b12n02:143969] Signal: Segmentation fault (11) +[b12n02:143969] Signal code: (3) +[b12n02:143969] Failing at address: 0xb170000000000a0 +[b12n02:143969] [ 0] [0x1000000504d8] +[b12n02:143969] [ 1] [0x3fffd0abe0f0] +[b12n02:143969] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n02:143969] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n02:143969] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n09:121753] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n09:121753] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n09:121753] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n09:121753] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n09:121753] [ 9] [b22n13:122034] *** Process received signal *** +[b22n13:122034] Signal: Segmentation fault (11) +[b22n13:122034] Signal code: (3) +[b22n13:122034] Failing at address: 0x14170000000000a0 +[b22n13:122034] [ 0] [0x1000000504d8] +[b22n13:122034] [ 1] [0x3fffc1db0910] +[b22n13:122034] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n13:122034] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n13:122034] [ 4] [b17n01:124279] *** Process received signal *** +[b17n01:124279] Signal: Segmentation fault (11) +[b17n01:124279] Signal code: (3) +[b17n01:124279] Failing at address: 0x14170000000000a0 +[b17n01:124279] [ 0] [0x1000000504d8] +[b17n01:124279] [ 1] [0x3fffc6171a60] +[b17n01:124279] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n01:124279] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n01:124279] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n01:124279] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n09:119633] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n09:119633] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n09:119633] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n09:119633] [ 8] [b11n14:161621] *** Process received signal *** +[b11n14:161621] Signal: Segmentation fault (11) +[b11n14:161621] Signal code: (3) +[b11n14:161621] Failing at address: 0x14170000000000a0 +[b11n14:161621] [ 0] [0x1000000504d8] +[b11n14:161621] [ 1] [0x3fffd641b720] +[b11n14:161621] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n14:161621] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n01:165128] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n01:165128] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n01:165128] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n01:165128] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n01:165128] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n18:140385] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n18:140385] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n18:140385] [ 6] [b16n03:121811] *** Process received signal *** +[b16n03:121811] Signal: Segmentation fault (11) +[b16n03:121811] Signal code: (3) +[b16n03:121811] Failing at address: 0xb170000000000a0 +[b16n03:121811] [ 0] [0x1000000504d8] +[b16n03:121811] [ 1] [0x3fffee929020] +[b16n03:121811] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n03:121811] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n03:121811] [ 4] [b17n03:135039] *** Process received signal *** +[b17n03:135039] Signal: Segmentation fault (11) +[b17n03:135039] Signal code: (3) +[b17n03:135039] Failing at address: 0x5170000000000a0 +[b17n03:135039] [ 0] [0x1000000504d8] +[b17n03:135039] [ 1] [0x3fffcb056800] +[b17n03:135039] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n03:135039] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n03:135039] [ 4] [a02n04:41673] *** Process received signal *** +[a02n04:41673] Signal: Segmentation fault (11) +[a02n04:41673] Signal code: (3) +[a02n04:41673] Failing at address: 0x14170000000000a0 +[a02n04:41673] [ 0] [0x1000000504d8] +[a02n04:41673] [ 1] [0x3fffdb2081c0] +[a02n04:41673] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n04:41673] [ 3] [0x1000000504d8] +[a30n15:154384] [ 1] [0x3fffd1d0e7e0] +[a30n15:154384] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n15:154384] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n15:154384] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n09:155758] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n09:155758] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n09:155758] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n02:139028] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n02:139028] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n02:139028] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n02:139028] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n02:139028] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n02:139028] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n02:139028] [b23n05:120229] *** Process received signal *** +[b23n05:120229] Signal: Segmentation fault (11) +[b23n05:120229] Signal code: (3) +[b23n05:120229] Failing at address: 0x14170000000000a0 +[b23n05:120229] [ 0] [0x1000000504d8] +[b23n05:120229] [ 1] [b06n12:177121] *** Process received signal *** +[b06n12:177121] Signal: Segmentation fault (11) +[b06n12:177121] Signal code: (3) +[b06n12:177121] Failing at address: 0x14170000000000a0 +[b06n12:177121] [ 0] [0x1000000504d8] +[b06n12:177121] [ 1] [0x3fffc863ed30] +[b06n12:177121] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n12:177121] [ 3] [b02n10:01251] *** Process received signal *** +[b02n10:01251] Signal: Segmentation fault (11) +[b02n10:01251] Signal code: (3) +[b02n10:01251] Failing at address: 0x14170000000000a0 +[b02n10:01251] [ 0] [0x1000000504d8] +[b02n10:01251] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n07:60174] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n07:60174] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n07:60174] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n07:60174] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n07:60174] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n16:124666] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n16:124666] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n16:124666] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n16:124666] [14] [a31n16:162627] [ 0] [0x1000000504d8] +[a31n16:162627] [ 1] [0x3fffce0a5f60] +[a31n16:162627] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n16:162627] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n16:162627] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n11:176584] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n11:176584] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n11:176584] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n11:176584] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n11:176584] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n11:176584] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n11:176584] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n11:176584] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n11:176584] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n10:176262] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n10:176262] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n10:176262] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n10:176262] [10] [a35n11:148676] *** Process received signal *** +[a35n11:148676] Signal: Segmentation fault (11) +[a35n11:148676] Signal code: (3) +[a35n11:148676] Failing at address: 0x5170000000000a0 +[a35n11:148676] [ 0] [0x1000000504d8] +[a35n11:148676] [ 1] [0x3fffc06121e0] +[a35n11:148676] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n11:148676] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n11:148676] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n11:171966] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n11:171966] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n11:171966] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n11:171966] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n11:171966] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n11:171966] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n11:171966] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n11:171966] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n11:171966] [12] [b13n03:126188] *** Process received signal *** +[b13n03:126188] Signal: Segmentation fault (11) +[b13n03:126188] Signal code: (3) +[b13n03:126188] Failing at address: 0xb170000000000a0 +[b13n03:126188] [ 0] [0x1000000504d8] +[b13n03:126188] [ 1] [0x3ffff6804c40] +[b13n03:126188] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n03:126188] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n14:175713] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n14:175713] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n14:175713] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n12:122968] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n12:122968] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n12:122968] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n12:122968] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n12:122968] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n04:166209] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n04:166209] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n04:166209] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n12:167676] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n12:167676] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n12:167676] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n12:167676] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n12:167676] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n12:167676] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n12:167676] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n12:167676] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n12:167676] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n12:167676] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n12:167676] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n12:167676] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n12:167676] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n12:180118] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n12:180118] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n12:180118] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n12:138736] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n12:138736] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n12:138736] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n15:123042] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n15:123042] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n15:123042] [ 6] [a26n04:54380] *** Process received signal *** +[a26n04:54380] Signal: Segmentation fault (11) +[a26n04:54380] Signal code: (3) +[a26n04:54380] Failing at address: 0xb170000000000a0 +[a26n04:54380] [ 0] [0x1000000504d8] +[a26n04:54380] [ 1] [0x3ffffe4a3830] +[a26n04:54380] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n04:54380] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n04:54380] [ 4] [b21n12:120132] *** Process received signal *** +[b21n12:120132] Signal: Segmentation fault (11) +[b21n12:120132] Signal code: (3) +[b21n12:120132] Failing at address: 0x14170000000000a0 +[b21n12:120132] [ 0] [0x1000000504d8] +[b21n12:120132] [ 1] [0x3fffeff1bac0] +[b21n12:120132] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n12:120132] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n12:120132] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n12:120132] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n12:120132] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n12:120132] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n12:120132] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n12:120132] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n12:120132] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n12:120132] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n12:120132] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n12:120132] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n12:120132] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n12:120132] *** End of error message *** +[b12n01:145732] *** Process received signal *** +[b12n01:145732] Signal: Segmentation fault (11) +[b12n01:145732] Signal code: (3) +[b12n01:145732] Failing at address: 0xb170000000000a0 +[b12n01:145732] [a36n18:163111] *** Process received signal *** +[a36n18:163111] Signal: Segmentation fault (11) +[a36n18:163111] Signal code: (3) +[a36n18:163111] Failing at address: 0x14170000000000a0 +[a36n18:163111] [ 0] [0x1000000504d8] +[a36n18:163111] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n10:126195] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n10:126195] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n10:126195] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n10:126195] [14] [a34n09:149101] *** Process received signal *** +[a34n09:149101] Signal: Segmentation fault (11) +[a34n09:149101] Signal code: (3) +[a34n09:149101] Failing at address: 0xb170000000000a0 +[a34n09:149101] [ 0] [0x1000000504d8] +[a34n09:149101] [ 1] [0x3ffff2951f50] +[a34n09:149101] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n09:149101] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n09:149101] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n12:154346] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n12:154346] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n12:154346] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n12:154346] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n12:154346] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n12:154346] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n12:154346] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n12:154346] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n12:154346] [13] [a26n12:52455] *** Process received signal *** +[a26n12:52455] Signal: Segmentation fault (11) +[a26n12:52455] Signal code: (3) +[a26n12:52455] Failing at address: 0xb170000000000a0 +[a26n12:52455] [ 0] [0x1000000504d8] +[a26n12:52455] [ 1] [0x3fffec11f510] +[a26n12:52455] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n12:52455] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n12:52455] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n12:52455] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n12:52455] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n01:141342] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n01:141342] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n01:141342] [14] [b15n17:126525] *** Process received signal *** +[b15n17:126525] Signal: Segmentation fault (11) +[b15n17:126525] Signal code: (3) +[b15n17:126525] Failing at address: 0x5170000000000a0 +[b15n17:126525] [ 0] [0x1000000504d8] +[b15n17:126525] [ 1] [0x3fffe97ab330] +[b15n17:126525] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n17:126525] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n17:171202] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n17:171202] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n17:171202] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n17:171202] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n10:122758] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n10:122758] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n10:122758] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n10:130584] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n10:130584] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n10:130584] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n10:130584] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n14:139844] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n14:139844] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n14:139844] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n14:139844] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n02:176052] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n02:176052] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n02:176052] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n02:176052] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n02:176052] [ 9] [a34n17:149904] *** Process received signal *** +[a34n17:149904] Signal: Segmentation fault (11) +[a34n17:149904] Signal code: (3) +[a34n17:149904] Failing at address: 0x14170000000000a0 +[a34n17:149904] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n06:177008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n06:177008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n06:177008] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n06:177008] [11] [b03n13:01301] *** Process received signal *** +[b03n13:01301] Signal: Segmentation fault (11) +[b03n13:01301] Signal code: (3) +[b03n13:01301] Failing at address: 0x14170000000000a0 +[b03n13:01301] [ 0] [0x1000000504d8] +[b03n13:01301] [ 1] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n14:133967] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n14:133967] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n14:133967] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n14:51452] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n14:51452] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n14:51452] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n14:51452] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n14:51452] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n14:51452] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n14:51452] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n14:51452] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n14:51452] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n14:51452] [13] [b17n06:119116] *** Process received signal *** +[b17n06:119116] Signal: Segmentation fault (11) +[b17n06:119116] Signal code: (3) +[b17n06:119116] Failing at address: 0x14170000000000a0 +[b17n06:119116] [ 0] [0x1000000504d8] +[b17n06:119116] [ 1] [0x3fffe9ace1b0] +[b17n06:119116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n06:119116] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n06:119116] [ 4] [0x1000000504d8] +[b08n16:171921] [ 1] [0x3ffff78aa7b0] +[b08n16:171921] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n16:171921] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n16:171921] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n16:171921] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n13:06926] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n13:06926] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n13:06926] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n13:06926] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n11:134750] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n11:134750] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n11:134750] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n11:134750] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n11:134750] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n11:134750] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n15:148584] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n15:148584] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n15:148584] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n15:148584] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n10:164938] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n10:164938] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n10:164938] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n10:164938] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n15:177857] [ 6] [b08n12:12766] *** Process received signal *** +[b08n12:12766] Signal: Segmentation fault (11) +[b08n12:12766] Signal code: (3) +[b08n12:12766] Failing at address: 0xb170000000000a0 +[b08n12:12766] [ 0] [0x1000000504d8] +[b08n12:12766] [ 1] [0x3fffe363e670] +[b08n12:12766] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n12:12766] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n12:12766] [ 4] [b15n08:118552] [ 0] [0x1000000504d8] +[b15n08:118552] [ 1] [0x3fffdc04ba70] +[b15n08:118552] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n08:118552] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n13:122189] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n13:122189] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n13:122189] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n13:122189] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n05:53374] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n05:53374] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n05:53374] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n07:173045] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n07:173045] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n07:173045] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n07:173045] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n06:133954] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n06:133954] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n06:133954] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n06:133954] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n09:128001] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n09:128001] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n09:128001] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n09:128001] [14] [b09n18:164801] *** Process received signal *** +[b09n18:164801] Signal: Segmentation fault (11) +[b09n18:164801] Signal code: (3) +[b09n18:164801] Failing at address: 0xb170000000000a0 +[b09n18:164801] [ 0] [0x1000000504d8] +[b09n18:164801] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n09:123420] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n09:123420] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n09:123420] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n09:123420] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n08:175809] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n08:175809] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n08:175809] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n08:175809] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n11:02687] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n11:02687] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n09:170659] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n09:170659] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n09:170659] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n09:170659] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n08:54628] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n08:54628] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n08:54628] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n08:54628] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n08:54628] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n08:54628] [14] [b11n06:164017] *** Process received signal *** +[b11n06:164017] Signal: Segmentation fault (11) +[b11n06:164017] Signal code: (3) +[b11n06:164017] Failing at address: 0x14170000000000a0 +[b11n06:164017] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n10:67859] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n10:67859] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n03:135166] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n03:135166] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n03:135166] [a32n04:153964] *** Process received signal *** +[a32n04:153964] Signal: Segmentation fault (11) +[a32n04:153964] Signal code: (3) +[a32n04:153964] Failing at address: 0xb170000000000a0 +[a32n04:153964] [b13n13:133628] *** Process received signal *** +[b13n13:133628] Signal: Segmentation fault (11) +[b13n13:133628] Signal code: (3) +[b13n13:133628] Failing at address: 0x14170000000000a0 +[b13n13:133628] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n18:176811] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n18:176811] [ 6] [0x1000000504d8] +[b09n13:164692] [ 1] [0x3ffffd3073b0] +[b09n13:164692] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n13:164692] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n13:164692] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n13:164692] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n13:164692] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n17:119979] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n17:119979] [13] [a27n15:32402] *** Process received signal *** +[a27n15:32402] Signal: Segmentation fault (11) +[a27n15:32402] Signal code: (3) +[a27n15:32402] Failing at address: 0x14170000000000a0 +[a27n15:32402] [ 0] [0x1000000504d8] +[a27n15:32402] [ 1] [0x3ffff9f716c0] +[a27n15:32402] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n15:32402] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n17:08309] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n17:08309] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n17:08309] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n17:08309] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n17:08309] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n17:08309] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n17:08309] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n17:08309] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n17:08309] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n16:166691] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n16:166691] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n16:166691] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n16:166691] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n16:166691] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n16:166691] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n16:166691] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n16:166691] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n16:166691] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n17:118576] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n13:54243] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n13:54243] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n13:54243] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n13:54243] [10] [b18n05:135124] *** Process received signal *** +[b18n05:135124] Signal: Segmentation fault (11) +[b18n05:135124] Signal code: (3) +[b18n05:135124] Failing at address: 0xb170000000000a0 +[b18n05:135124] [ 0] [b22n07:120699] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n07:120699] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n07:120699] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n07:120699] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n07:120699] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n07:120699] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n07:120699] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n07:120699] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n07:120699] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n07:120699] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n11:117474] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n11:117474] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n11:117474] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n11:117474] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n11:117474] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n11:117474] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n11:117474] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n14:173897] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n14:173897] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n14:173897] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n14:173897] [12] [a29n11:162471] *** Process received signal *** +[a29n11:162471] Signal: Segmentation fault (11) +[a29n11:162471] Signal code: (3) +[a29n11:162471] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n18:177224] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n18:177224] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n18:177224] [12] [b07n13:165260] *** Process received signal *** +[b07n13:165260] Signal: Segmentation fault (11) +[b07n13:165260] Signal code: (3) +[b07n13:165260] Failing at address: 0x14170000000000a0 +[b07n13:165260] [ 0] [0x1000000504d8] +[b07n13:165260] [ 1] [0x3fffff31eda0] +[b07n13:165260] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n13:165260] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n02:167600] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n02:167600] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n02:167600] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n02:167600] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n12:119807] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n12:119807] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n12:119807] [ 6] [b21n14:119485] *** Process received signal *** +[b21n14:119485] Signal: Segmentation fault (11) +[b21n14:119485] Signal code: (3) +[b21n14:119485] Failing at address: 0x14170000000000a0 +[b21n14:119485] [ 0] [0x1000000504d8] +[b21n14:119485] [ 1] [0x3fffdcfb3270] +[b21n14:119485] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n16:173551] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n16:173551] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n16:173551] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n16:173551] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n16:173551] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n16:173551] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n16:173551] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n16:173551] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n16:173551] *** End of error message *** +[10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n16:147285] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n16:147285] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n16:147285] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n16:147285] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n16:147285] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n13:168160] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n13:168160] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n13:168160] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n13:168160] [14] [a31n02:158421] *** Process received signal *** +[a31n02:158421] Signal: Segmentation fault (11) +[a31n02:158421] Signal code: (3) +[a31n02:158421] Failing at address: 0x14170000000000a0 +[a31n02:158421] [ 0] [0x1000000504d8] +[a31n02:158421] [ 1] [0x3fffea7fc3e0] +[a31n02:158421] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n02:158421] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n11:153417] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n11:153417] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n11:153417] [ 9] [b14n01:128003] *** Process received signal *** +[b14n01:128003] Signal: Segmentation fault (11) +[b14n01:128003] Signal code: (3) +[b14n01:128003] Failing at address: 0xb170000000000a0 +[b14n01:128003] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n11:170899] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n11:170899] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n08:116176] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n08:116176] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n08:116904] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n08:116904] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n18:138474] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n16:166500] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n16:166500] *** End of error message *** +[0x1000000504d8] +[a28n05:04519] [ 1] [0x3fffe78e3050] +[a28n05:04519] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n05:04519] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n05:04519] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n05:04519] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n05:04519] [ 6] [0x1000000504d8] +[b15n12:132104] [ 1] [0x3fffcdb2ed10] +[b15n12:132104] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n12:132104] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n17:176829] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n17:176829] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n17:176829] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n17:176829] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n15:139748] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n15:139748] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n13:117509] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n13:117509] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n13:117509] [ 9] [b02n02:166745] *** Process received signal *** +[b02n02:166745] Signal: Segmentation fault (11) +[b02n02:166745] Signal code: (3) +[b02n02:166745] Failing at address: 0x14170000000000a0 +[b02n02:166745] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n04:44760] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n04:44760] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n07:117363] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n07:117363] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n07:117363] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n07:117363] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n17:163820] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n17:163820] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n07:174563] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n07:174563] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n07:174563] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n07:174563] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n16:01574] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n16:01574] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n16:01574] [ 6] [a27n04:52703] [ 0] [0x1000000504d8] +[a27n04:52703] [ 1] [0x3fffe9d96830] +[a27n04:52703] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n04:52703] [ 3] [b06n16:165922] *** Process received signal *** +[b06n16:165922] Signal: Segmentation fault (11) +[b06n16:165922] Signal code: (3) +[b06n16:165922] Failing at address: 0x14170000000000a0 +[b06n16:165922] [ 0] [0x1000000504d8] +[b06n16:165922] [ 1] [0x3fffd47dbc30] +[b06n16:165922] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n16:165922] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n09:51335] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n09:51335] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n09:51335] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n17:119469] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n17:119469] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n17:119469] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n17:119469] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n01:153684] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n01:153684] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n01:153684] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n01:153684] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n06:130545] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n06:130545] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n06:130545] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n06:130545] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n06:130545] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n06:130545] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n06:130545] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n06:130545] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n06:130545] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n17:138614] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n17:138614] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n17:138614] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n17:138614] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n17:138614] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n17:138614] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n17:138614] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n17:138614] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n17:138614] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n17:138614] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n18:134185] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n18:134185] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n18:134185] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n18:134185] [ 8] [b09n10:169682] *** Process received signal *** +[b09n10:169682] Signal: Segmentation fault (11) +[b09n10:169682] Signal code: (3) +[b09n10:169682] Failing at address: 0xb170000000000a0 +[b09n10:169682] [ 0] [0x1000000504d8] +[b09n10:169682] [ 1] [0x3fffe7734ec0] +[b09n10:169682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n10:169682] [ 3] [0x1000000504d8] +[b14n03:130350] [ 1] [0x3fffc8bc3820] +[b14n03:130350] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n03:130350] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n03:130350] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n12:156919] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n12:156919] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n12:156919] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n12:156919] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n12:156919] *** End of error message *** +[b12n17:138143] *** Process received signal *** +[b12n17:138143] Signal: Segmentation fault (11) +[b12n17:138143] Signal code: (3) +[b12n17:138143] Failing at address: 0xb170000000000a0 +[b12n17:138143] [ 0] [0x1000000504d8] +[b12n17:138143] [ 1] [0x3fffe61dbb30] +[b12n17:138143] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n17:138143] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n17:138143] [ 4] [a34n01:164962] *** Process received signal *** +[a34n01:164962] Signal: Segmentation fault (11) +[a34n01:164962] Signal code: (3) +[a34n01:164962] Failing at address: 0xb170000000000a0 +[a34n01:164962] [ 0] [0x1000000504d8] +[a34n01:164962] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n18:69152] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n06:167094] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n06:167094] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n12:140125] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n12:140125] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n12:140125] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n12:140125] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n01:178525] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n01:178525] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n01:178525] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n01:178525] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n01:178525] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n01:178525] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n01:178525] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n01:178525] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n01:178525] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n01:178525] [13] [b01n18:179701] *** Process received signal *** +[b01n18:179701] Signal: Segmentation fault (11) +[b01n18:179701] Signal code: (3) +[b01n18:179701] Failing at address: 0x5170000000000a0 +[b01n18:179701] [ 0] [0x1000000504d8] +[b01n18:179701] [ 1] [0x3fffc0e1fbb0] +[b01n18:179701] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n18:179701] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n03:09643] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n14:126123] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n14:126123] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n14:126123] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n17:124168] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n17:124168] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n17:124168] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n17:124168] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n17:124168] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n02:125268] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n02:125268] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n02:125268] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n02:125268] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n02:125268] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n02:125268] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n02:125268] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n02:125268] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n14:177226] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n14:177226] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n14:177226] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n14:177226] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n14:177226] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n14:177226] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n14:177226] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n14:177226] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n14:177226] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n07:153179] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n07:153179] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n07:153179] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n07:153179] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n07:153179] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n07:153179] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n07:153179] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n07:153179] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n07:153179] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n07:153179] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n11:124433] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n11:124433] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n11:124433] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n11:124433] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n11:124433] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n11:124433] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n15:148165] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n15:148165] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n15:148165] [ 8] [a33n06:165306] *** Process received signal *** +[a33n06:165306] Signal: Segmentation fault (11) +[a33n06:165306] Signal code: (3) +[a33n06:165306] Failing at address: 0x14170000000000a0 +[a33n06:165306] [ 0] [0x1000000504d8] +[a33n06:165306] [ 1] [0x3fffc6c6a090] +[a33n06:165306] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n06:165306] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n06:165306] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n01:48807] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n01:48807] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n01:48807] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n01:48807] [14] [b08n11:164998] *** Process received signal *** +[b08n11:164998] Signal: Segmentation fault (11) +[b08n11:164998] Signal code: (3) +[b08n11:164998] Failing at address: 0x14170000000000a0 +[b08n11:164998] [ 0] [0x1000000504d8] +[b08n11:164998] [ 1] [0x3fffc826beb0] +[b08n11:164998] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n11:164998] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n11:164998] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n01:132064] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n01:132064] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n01:132064] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n14:176071] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n14:176071] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n14:176071] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n14:176071] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n13:10200] *** Process received signal *** +[b05n13:10200] Signal: Segmentation fault (11) +[b05n13:10200] Signal code: (3) +[b05n13:10200] Failing at address: 0x14170000000000a0 +[b05n13:10200] [ 0] [0x1000000504d8] +[b05n13:10200] [ 1] [0x3ffffa39a010] +[b05n13:10200] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n13:10200] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n07:133374] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n07:133374] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n07:133374] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n07:133374] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n07:133374] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n13:177639] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n13:177639] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n13:177639] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n13:177639] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n13:177639] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n13:177639] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n13:177639] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n13:177639] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n13:177639] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n13:177639] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n13:177639] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n13:177639] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n09:03930] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n09:03930] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n09:03930] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n09:03930] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n09:03930] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n09:03930] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n09:03930] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n09:03930] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n09:03930] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n09:03930] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n09:03930] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n04:138157] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n04:138157] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n04:138157] [13] [b18n12:120406] *** Process received signal *** +[b18n12:120406] Signal: Segmentation fault (11) +[b18n12:120406] Signal code: (3) +[b18n12:120406] Failing at address: 0x14170000000000a0 +[b18n12:120406] [ 0] [0x1000000504d8] +[b18n12:120406] [ 1] [0x3fffce233b40] +[b18n12:120406] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n12:120406] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n12:120406] [ 4] [b22n15:120062] *** Process received signal *** +[b22n15:120062] Signal: Segmentation fault (11) +[b22n15:120062] Signal code: (3) +[b22n15:120062] Failing at address: 0x14170000000000a0 +[b22n15:120062] [ 0] [0x1000000504d8] +[b22n15:120062] [ 1] [0x3ffffdda8ce0] +[b22n15:120062] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n15:120062] [ 3] [b02n13:171108] *** Process received signal *** +[b02n13:171108] Signal: Segmentation fault (11) +[b02n13:171108] Signal code: (3) +[b02n13:171108] Failing at address: 0xb170000000000a0 +[b02n13:171108] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n16:122493] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n02:162034] *** Process received signal *** +[b08n02:162034] Signal: Segmentation fault (11) +[b08n02:162034] Signal code: (3) +[b08n02:162034] Failing at address: 0x14170000000000a0 +[b08n02:162034] [ 0] [0x1000000504d8] +[b08n02:162034] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n11:123523] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n11:123523] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n15:177056] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n15:177056] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n15:177056] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n15:177056] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n15:177056] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n15:177056] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n10:155398] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n10:155398] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n10:155398] [12] [a36n13:156922] *** Process received signal *** +[a36n13:156922] Signal: Segmentation fault (11) +[a36n13:156922] Signal code: (3) +[a36n13:156922] Failing at address: 0xb170000000000a0 +[a36n13:156922] [ 0] [0x1000000504d8] +[a36n13:156922] [ 1] [0x3ffffef10ad0] +[a36n13:156922] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n13:156922] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n13:156922] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n08:165286] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n08:165286] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n08:165286] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n08:165286] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n15:169492] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n15:169492] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n05:149467] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n05:149467] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n13:137749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n13:137749] *** End of error message *** +[0x1000000504d8] +[a33n05:139438] [ 1] [0x3fffed7690c0] +[a33n05:139438] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n05:139438] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n05:139438] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n05:139438] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n05:135167] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n05:135167] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n05:135167] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n05:135167] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n05:135167] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n17:179117] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n17:179117] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n17:179117] *** End of error message *** +[b18n03:136669] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n03:136669] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n03:136669] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n03:136669] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n16:127468] *** Process received signal *** +[b16n16:127468] Signal: Segmentation fault (11) +[b16n16:127468] Signal code: (3) +[b16n16:127468] Failing at address: 0x14170000000000a0 +[b16n16:127468] [ 0] [0x1000000504d8] +[b16n16:127468] [ 1] [0x3ffff271bad0] +[b16n16:127468] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n16:127468] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n15:132616] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n15:132616] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n15:132616] [14] [b14n12:123563] *** Process received signal *** +[b14n12:123563] Signal: Segmentation fault (11) +[b14n12:123563] Signal code: (3) +[b14n12:123563] Failing at address: 0xb170000000000a0 +[b14n12:123563] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n07:172881] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n07:172881] [ 6] [a36n10:142934] *** Process received signal *** +[a36n10:142934] Signal: Segmentation fault (11) +[a36n10:142934] Signal code: (3) +[a36n10:142934] Failing at address: 0x14170000000000a0 +[a36n10:142934] [ 0] [0x1000000504d8] +[a36n10:142934] [ 1] [0x3ffff7d60470] +[a36n10:142934] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n10:142934] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n10:142934] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n10:142934] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n10:142934] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n10:142934] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n10:142934] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n10:142934] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n10:142934] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n10:142934] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n10:142934] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n10:142934] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n10:142934] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n10:142934] *** End of error message *** +[0x3fffe10c5fa0] +[b11n10:161718] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n10:161718] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n10:161718] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n10:161718] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n10:161718] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n07:116116] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n07:116116] *** End of error message *** +[b08n05:173848] *** Process received signal *** +[b08n05:173848] Signal: Segmentation fault (11) +[b08n05:173848] Signal code: (3) +[b08n05:173848] Failing at address: 0xb170000000000a0 +[b08n05:173848] [ 0] [0x1000000504d8] +[b08n05:173848] [ 1] [0x3fffe7486dc0] +[b08n05:173848] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n05:173848] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n05:173848] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n12:145267] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n12:145267] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n17:162359] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n17:162359] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n17:162359] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n17:162359] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n17:162359] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n17:162359] *** End of error message *** +[a31n01:173783] *** Process received signal *** +[a31n01:173783] Signal: Segmentation fault (11) +[a31n01:173783] Signal code: (3) +[a31n01:173783] Failing at address: 0x14170000000000a0 +[a31n01:173783] [ 0] [0x1000000504d8] +[a31n01:173783] [ 1] [0x3fffe78bdc50] +[a31n01:173783] [ 2] [a28n10:176308] *** Process received signal *** +[a28n10:176308] Signal: Segmentation fault (11) +[a28n10:176308] Signal code: (3) +[a28n10:176308] Failing at address: 0xb170000000000a0 +[a28n10:176308] [ 0] [0x1000000504d8] +[a28n10:176308] [ 1] [0x3fffda5b7fe0] +[a28n10:176308] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n10:176308] [ 3] [b17n13:123518] *** Process received signal *** +[b17n13:123518] Signal: Segmentation fault (11) +[b17n13:123518] Signal code: (3) +[b17n13:123518] Failing at address: 0x14170000000000a0 +[b17n13:123518] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n15:03735] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n15:03735] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n15:03735] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n15:03735] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n15:03735] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n15:03735] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n15:03735] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n15:03735] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n15:03735] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n15:03735] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n01:171789] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n01:171789] *** End of error message *** +[b22n16:124796] *** Process received signal *** +[b22n16:124796] Signal: Segmentation fault (11) +[b22n16:124796] Signal code: (3) +[b22n16:124796] Failing at address: 0x14170000000000a0 +[b22n16:124796] [ 0] [0x1000000504d8] +[b22n16:124796] [ 1] [0x3fffeb6ea110] +[b22n16:124796] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n16:124796] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n16:124796] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n14:128872] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n14:128872] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n14:128872] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n14:128872] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n14:128872] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n01:165164] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n01:165164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n01:165164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n01:165164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n01:165164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n01:165164] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n01:165164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n01:165164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n01:165164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n01:165164] [14] [b05n10:10924] *** Process received signal *** +[b05n10:10924] Signal: Segmentation fault (11) +[b05n10:10924] Signal code: (3) +[b05n10:10924] Failing at address: 0xb170000000000a0 +[b05n10:10924] [ 0] [0x1000000504d8] +[b05n10:10924] [ 1] [0x3fffec96b3f0] +[b05n10:10924] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n10:10924] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n15:176427] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n07:118165] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n07:118165] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n07:118165] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n07:132088] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n07:132088] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n07:132088] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n07:132088] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n18:131586] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n18:131586] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n01:163392] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n01:163392] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n01:163392] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n01:163392] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n07:156141] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n07:156141] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n07:156141] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n07:156141] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n06:154621] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n06:154621] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n06:154621] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n06:154621] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n01:179264] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n01:179264] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n01:179264] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n06:171502] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n06:171502] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n06:171502] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n06:171502] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n14:177514] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n14:177514] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n14:177514] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n14:177514] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n14:177514] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n14:177514] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n14:177514] [10] [a30n17:176415] *** Process received signal *** +[a30n17:176415] Signal: Segmentation fault (11) +[a30n17:176415] Signal code: (3) +[a30n17:176415] Failing at address: 0x14170000000000a0 +[a30n17:176415] [ 0] [0x1000000504d8] +[a30n17:176415] [ 1] [0x3fffcb831ce0] +[a30n17:176415] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n17:176415] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n17:176415] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n08:138179] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n08:138179] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n16:128440] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n16:128440] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n16:128440] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n16:128440] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n11:147714] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n11:147714] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n11:147714] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n11:147714] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n01:167302] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n01:167302] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n12:123235] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n12:123235] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n12:123235] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n12:123235] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n12:123235] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n05:163965] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n05:163965] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n04:117356] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n04:117356] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n04:117356] [ 6] [0x1000000504d8] +[b14n11:130805] [ 1] [0x3fffe3e6f5b0] +[b14n11:130805] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n11:130805] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n11:130805] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n11:130805] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n17:122357] *** End of error message *** +[a32n12:164384] *** Process received signal *** +[a32n12:164384] Signal: Segmentation fault (11) +[a32n12:164384] Signal code: (3) +[a32n12:164384] Failing at address: 0x14170000000000a0 +[a32n12:164384] [ 0] [0x1000000504d8] +[a32n12:164384] [ 1] [0x3fffc2cdadf0] +[a32n12:164384] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n12:164384] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n12:164384] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n05:139911] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n05:139911] *** End of error message *** +[b11n05:162570] *** Process received signal *** +[b11n05:162570] Signal: Segmentation fault (11) +[b11n05:162570] Signal code: (3) +[b11n05:162570] Failing at address: 0x14170000000000a0 +[b11n05:162570] [ 0] [b10n11:160351] *** Process received signal *** +[b10n11:160351] Signal: Segmentation fault (11) +[b10n11:160351] Signal code: (3) +[b10n11:160351] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n08:164750] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n08:164750] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n08:164750] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n13:165389] *** Process received signal *** +[a29n13:165389] Signal: Segmentation fault (11) +[a29n13:165389] Signal code: (3) +[a29n13:165389] Failing at address: 0x14170000000000a0 +[a29n13:165389] [ 0] [0x1000000504d8] +[a29n13:165389] [ 1] [0x3fffd39a1f30] +[a29n13:165389] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n13:165389] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n13:138092] *** End of error message *** +[a27n18:28218] [ 0] [0x1000000504d8] +[a27n18:28218] [ 1] [0x3fffe9cfa6a0] +[a27n18:28218] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n18:28218] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n06:127274] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n06:127274] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n06:127274] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n08:163165] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n08:163165] [ 6] [b01n10:173680] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n03:120669] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n03:120669] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n03:120669] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n03:120669] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n03:120669] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n03:120669] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n03:120669] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n03:120669] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n03:120669] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n03:120669] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n03:120669] *** End of error message *** +[b08n15:177821] *** Process received signal *** +[b08n15:177821] Signal: Segmentation fault (11) +[b08n15:177821] Signal code: (3) +[b08n15:177821] Failing at address: 0x14170000000000a0 +[b08n15:177821] [ 0] [0x1000000504d8] +[b08n15:177821] [ 1] [0x3fffd11136c0] +[b08n15:177821] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n15:177821] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n15:177821] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n13:114399] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n13:114399] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n13:114399] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n13:114399] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n13:114399] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n13:114399] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n10:162609] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n10:162609] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n10:162609] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n14:152041] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n14:152041] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n14:152041] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n02:164490] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n02:164490] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n02:164490] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n02:149228] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n02:149228] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n02:149228] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n02:149228] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n02:149228] *** End of error message *** +[a31n14:165661] *** Process received signal *** +[a31n14:165661] Signal: Segmentation fault (11) +[a31n14:165661] Signal code: (3) +[a31n14:165661] Failing at address: 0x14170000000000a0 +[a29n02:163030] *** Process received signal *** +[a29n02:163030] Signal: Segmentation fault (11) +[a29n02:163030] Signal code: (3) +[a29n02:163030] Failing at address: 0x5170000000000a0 +[a29n02:163030] [ 0] [b08n03:175064] *** Process received signal *** +[b08n03:175064] Signal: Segmentation fault (11) +[b08n03:175064] Signal code: (3) +[b08n03:175064] Failing at address: 0x14170000000000a0 +[b08n03:175064] [ 0] [0x1000000504d8] +[b08n03:175064] [ 1] [0x3fffe598b270] +[b08n03:175064] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n03:175064] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n03:175064] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n08:131381] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n08:131381] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n08:131381] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n08:131381] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n09:173754] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n09:173754] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n09:173754] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n09:173754] [10] [b08n13:158867] *** Process received signal *** +[b08n13:158867] Signal: Segmentation fault (11) +[b08n13:158867] Signal code: (3) +[b08n13:158867] Failing at address: 0x14170000000000a0 +[b08n13:158867] [ 0] [0x1000000504d8] +[b08n13:158867] [ 1] [0x3fffd788b980] +[b08n13:158867] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n13:158867] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n13:158867] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n17:119584] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n17:119584] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n10:131276] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n10:131276] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n10:131276] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n13:163196] [ 6] [b10n15:164413] *** Process received signal *** +[b10n15:164413] Signal: Segmentation fault (11) +[b10n15:164413] Signal code: (3) +[b10n15:164413] Failing at address: 0xb170000000000a0 +[b10n15:164413] [ 0] [a33n04:169914] *** Process received signal *** +[a33n04:169914] Signal: Segmentation fault (11) +[a33n04:169914] Signal code: (3) +[a33n04:169914] Failing at address: 0x14170000000000a0 +[a33n04:169914] [ 0] [0x1000000504d8] +[a33n04:169914] [ 1] [0x3ffffa522910] +[a33n04:169914] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n04:169914] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n04:169914] [ 4] [b15n18:08432] *** Process received signal *** +[b15n18:08432] Signal: Segmentation fault (11) +[b15n18:08432] Signal code: (3) +[b15n18:08432] Failing at address: 0xb170000000000a0 +[b15n18:08432] [ 0] [0x1000000504d8] +[b15n18:08432] [ 1] [0x3fffd8bbf000] +[b15n18:08432] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n18:08432] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n18:08432] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n13:176365] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n13:176365] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n13:176365] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n13:176365] [10] [0x3fffd7a0ce20] +[b21n11:120585] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n11:120585] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n11:120585] [ 4] [b14n09:119547] *** Process received signal *** +[b14n09:119547] Signal: Segmentation fault (11) +[b14n09:119547] Signal code: (3) +[b14n09:119547] Failing at address: 0xb170000000000a0 +[b14n09:119547] [ 0] [0x1000000504d8] +[b14n09:119547] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n13:123053] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n13:123053] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n13:123053] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n18:21770] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n18:21770] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n18:21770] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n18:21770] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n18:21770] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n18:21770] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n18:21770] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n18:21770] [11] [0x1000000504d8] +[a31n08:156370] [ 1] [0x3fffe6064b00] +[a31n08:156370] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n08:156370] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n08:156370] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n02:178478] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n02:178478] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n02:178478] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n02:178478] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n02:178478] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n01:124279] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n01:124279] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n01:124279] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n01:124279] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n01:124279] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n15:120131] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n15:120131] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n15:120131] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n15:120131] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n03:135039] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n03:135039] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n03:135039] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n03:135039] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n03:135039] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n03:135039] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n03:135039] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n03:135039] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n03:135039] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n03:135039] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n04:41673] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n04:41673] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n04:41673] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n04:41673] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n04:41673] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n04:41673] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n04:41673] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n04:41673] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n04:41673] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n04:41673] [13] [b08n09:174689] *** Process received signal *** +[b08n09:174689] Signal: Segmentation fault (11) +[b08n09:174689] Signal code: (3) +[b08n09:174689] Failing at address: 0x14170000000000a0 +[b08n09:174689] [ 0] [0x1000000504d8] +[b08n09:174689] [ 1] [0x3ffff9e71860] +[b08n09:174689] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n09:174689] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n15:137006] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n15:137006] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n08:124724] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n08:124724] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n08:124724] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n08:124724] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n08:124724] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n08:124724] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n17:150932] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n18:163574] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n18:163574] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n18:163574] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n18:163574] [14] [0x1000000504d8] +[a35n01:138008] [ 1] [0x3fffe75e5740] +[a35n01:138008] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n01:138008] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n01:138008] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n12:177121] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n12:177121] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n12:177121] [ 6] [0x3ffffd654d90] +[b02n10:01251] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n10:01251] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n10:01251] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n10:01251] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n10:01251] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n10:01251] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n10:01251] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n10:01251] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n10:01251] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n10:01251] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n10:01251] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n10:01251] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n10:01251] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n10:01251] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n16:162627] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n11:176584] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n11:176584] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n11:176584] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n10:151894] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n10:151894] *** End of error message *** +[b16n05:122273] *** Process received signal *** +[b16n05:122273] Signal: Segmentation fault (11) +[b16n05:122273] Signal code: (3) +[b16n05:122273] Failing at address: 0x14170000000000a0 +[b16n05:122273] [ 0] [0x1000000504d8] +[b16n05:122273] [ 1] [0x3fffd221ba90] +[b16n05:122273] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n05:122273] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n05:122273] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n05:126780] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n05:126780] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n05:126780] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n05:126780] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n05:126780] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n05:126780] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n05:126780] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n05:126780] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n08:08954] *** Process received signal *** +[b05n08:08954] Signal: Segmentation fault (11) +[b05n08:08954] Signal code: (3) +[b05n08:08954] Failing at address: 0x14170000000000a0 +[b05n08:08954] [ 0] [0x1000000504d8] +[b05n08:08954] [ 1] [0x3ffff1720180] +[b05n08:08954] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n08:08954] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n08:08954] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n08:172432] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n08:172432] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n08:172432] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n08:172432] [13] [0x3fffde288130] +[b23n12:115641] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n12:115641] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n12:115641] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n12:115641] [ 5] [b08n06:161725] *** Process received signal *** +[b08n06:161725] Signal: Segmentation fault (11) +[b08n06:161725] Signal code: (3) +[b08n06:161725] Failing at address: 0x14170000000000a0 +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n15:151250] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n15:151250] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n12:180118] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n12:180118] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n04:54380] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n04:54380] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n04:54380] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n04:54380] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n04:54380] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n04:54380] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n04:54380] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n04:54380] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n04:54380] [13] [b08n14:178324] *** Process received signal *** +[b08n14:178324] Signal: Segmentation fault (11) +[b08n14:178324] Signal code: (3) +[b08n14:178324] Failing at address: 0xb170000000000a0 +[b08n14:178324] [ 0] [0x1000000504d8] +[b08n14:178324] [ 1] [0x3fffc08d2600] +[b08n14:178324] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n14:178324] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n18:169363] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n18:169363] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n18:169363] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n18:169363] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n18:169363] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n18:169363] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n08:127447] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n08:127447] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n08:127447] [11] [b11n15:166121] *** Process received signal *** +[b11n15:166121] Signal: Segmentation fault (11) +[b11n15:166121] Signal code: (3) +[b11n15:166121] Failing at address: 0x14170000000000a0 +[b11n15:166121] [ 0] [0x1000000504d8] +[b11n15:166121] [ 1] [0x3ffff953d3b0] +[b11n15:166121] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n15:166121] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n15:166121] [ 4] [a31n13:155852] *** Process received signal *** +[a31n13:155852] Signal: Segmentation fault (11) +[a31n13:155852] Signal code: (3) +[a31n13:155852] Failing at address: 0x14170000000000a0 +[a31n13:155852] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n12:52455] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n12:52455] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n12:52455] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n12:52455] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n12:52455] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n12:52455] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n12:52455] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n09:137776] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n09:137776] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n09:137776] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n09:137776] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n10:130581] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n10:130581] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n10:130581] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n10:130581] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n14:139844] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n14:139844] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n14:139844] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n14:139844] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n14:139844] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n02:176052] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n02:176052] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n02:176052] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n02:176052] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n15:165475] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n15:165475] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n15:165475] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n15:122086] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n15:122086] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n15:122086] [ 6] [b21n18:118685] *** Process received signal *** +[b21n18:118685] Signal: Segmentation fault (11) +[b21n18:118685] Signal code: (3) +[b21n18:118685] Failing at address: 0x14170000000000a0 +[b21n18:118685] [ 0] [0x1000000504d8] +[b21n18:118685] [ 1] [0x3fffd0f01e60] +[b21n18:118685] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n18:118685] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n18:118685] [ 4] [b18n07:125449] *** Process received signal *** +[b18n07:125449] Signal: Segmentation fault (11) +[b18n07:125449] Signal code: (3) +[b18n07:125449] Failing at address: 0x14170000000000a0 +[b18n07:125449] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n11:120240] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n11:120240] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n09:119633] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n09:119633] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n09:119633] [11] [a30n06:155089] *** Process received signal *** +[a30n06:155089] Signal: Segmentation fault (11) +[a30n06:155089] Signal code: (3) +[a30n06:155089] Failing at address: 0x14170000000000a0 +[a30n06:155089] [ 0] [0x1000000504d8] +[a30n06:155089] [ 1] [0x3fffd1c19a00] +[a30n06:155089] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n06:155089] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n11:152818] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n11:152818] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n11:152818] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n11:152818] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n11:152818] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n11:152818] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n11:152818] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n11:152818] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n11:152818] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n11:152818] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n09:140257] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n09:140257] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n09:140257] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n09:140257] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n09:140257] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n12:154343] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n12:154343] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n12:154343] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n12:154343] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n12:154343] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n12:154343] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n12:154343] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n12:154343] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n12:154343] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n12:154343] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n06:162917] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n06:162917] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n06:177008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n06:177008] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n06:177008] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n13:01302] *** Process received signal *** +[b03n13:01302] Signal: Segmentation fault (11) +[b03n13:01302] Signal code: (3) +[b03n13:01302] Failing at address: 0x14170000000000a0 +[b03n13:01302] [ 0] [0x1000000504d8] +[b03n13:01302] [ 1] [0x3fffd322d390] +[b03n13:01302] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n13:01302] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n14:51452] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n14:51452] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n06:119116] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n06:119116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n06:119116] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n16:171917] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n16:171917] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n13:06926] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n15:148584] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n15:148584] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n15:148584] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n15:148584] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n15:148584] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n15:148584] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n15:177857] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n15:177857] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n15:177857] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n15:177857] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n15:177857] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n15:177857] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n15:177857] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n07:138206] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n07:138206] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n07:138206] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n07:138206] [ 6] [a28n04:13082] *** Process received signal *** +[a28n04:13082] Signal: Segmentation fault (11) +[a28n04:13082] Signal code: (3) +[a28n04:13082] Failing at address: 0x14170000000000a0 +[a28n04:13082] [ 0] [0x1000000504d8] +[a28n04:13082] [ 1] [0x3fffff884720] +[a28n04:13082] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n04:13082] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n08:121435] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n08:121435] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n08:121435] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n08:121435] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n08:121435] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n08:121435] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n08:121435] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n08:121435] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n08:121435] [13] [b17n16:127351] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n09:173062] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n09:173062] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n09:173062] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n09:128001] *** End of error message *** +[0x3ffffe87ea10] +[b23n05:120229] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n05:120229] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n05:120229] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n05:120229] [ 5] [b11n09:161230] *** Process received signal *** +[b11n09:161230] Signal: Segmentation fault (11) +[b11n09:161230] Signal code: (3) +[b11n09:161230] Failing at address: 0x14170000000000a0 +[b11n09:161230] [ 0] [0x1000000504d8] +[b11n09:161230] [ 1] [0x3fffeed6abe0] +[b11n09:161230] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n09:161230] [ 3] [a29n12:160188] *** Process received signal *** +[a29n12:160188] Signal: Segmentation fault (11) +[a29n12:160188] Signal code: (3) +[a29n12:160188] Failing at address: 0x14170000000000a0 +[a29n12:160188] [ 0] [0x1000000504d8] +[a29n12:160188] [ 1] [a30n12:148380] *** Process received signal *** +[a30n12:148380] Signal: Segmentation fault (11) +[a30n12:148380] Signal code: (3) +[a30n12:148380] Failing at address: 0xb170000000000a0 +[a30n12:148380] [ 0] [0x1000000504d8] +[a30n12:148380] [ 1] [0x3fffff089cb0] +[a30n12:148380] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n12:148380] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n12:148380] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n08:54628] *** End of error message *** +[0x1000000504d8] +[b11n06:164016] [ 1] [0x3fffdadf9600] +[b11n06:164016] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n06:164016] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n06:164016] [ 4] [b03n07:03373] *** Process received signal *** +[b03n07:03373] Signal: Segmentation fault (11) +[b03n07:03373] Signal code: (3) +[b03n07:03373] Failing at address: 0xb170000000000a0 +[b03n07:03373] [ 0] [0x1000000504d8] +[b03n07:03373] [ 1] [0x3fffe162da10] +[b03n07:03373] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n07:03373] [ 3] [0x1000000504d8] +[b10n17:164564] [ 1] [0x3fffcf9c0850] +[b10n17:164564] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n17:164564] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n17:164564] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n17:164564] [ 5] [b03n05:05610] *** Process received signal *** +[b03n05:05610] Signal: Segmentation fault (11) +[b03n05:05610] Signal code: (3) +[b03n05:05610] Failing at address: 0x14170000000000a0 +[b03n05:05610] [ 0] [0x1000000504d8] +[b03n05:05610] [ 1] [0x3fffc57c9d70] +[b03n05:05610] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n05:05610] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n05:05610] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n09:121729] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n17:119979] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n17:119979] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n09:170659] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n18:176810] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n18:176810] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n18:176810] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n18:176810] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n18:176810] [10] [0x3ffff35c7260] +[a27n15:32401] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n15:32401] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n07:168965] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n07:168965] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n07:168965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n07:168965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n07:168965] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n13:54243] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n13:54243] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n13:54243] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n13:54243] [14] [b21n17:118573] *** Process received signal *** +[b21n17:118573] Signal: Segmentation fault (11) +[b21n17:118573] Signal code: (3) +[b21n17:118573] Failing at address: 0xb170000000000a0 +[b21n17:118573] [b22n08:118580] *** Process received signal *** +[b22n08:118580] Signal: Segmentation fault (11) +[b22n08:118580] Signal code: (3) +[b22n08:118580] Failing at address: 0x14170000000000a0 +[b22n08:118580] [ 0] [0x1000000504d8] +[b22n08:118580] [ 1] [0x3fffe639f2f0] +[b22n08:118580] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n08:118580] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n08:118580] [ 4] [b06n02:163320] *** Process received signal *** +[b06n02:163320] Signal: Segmentation fault (11) +[b06n02:163320] Signal code: (3) +[b06n02:163320] Failing at address: 0xb170000000000a0 +[b06n02:163320] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n18:177224] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n18:177224] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n18:177224] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n11:162474] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n11:162474] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n11:162474] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n13:165263] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n13:165263] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n13:165263] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n13:165263] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n13:165263] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n13:165263] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n14:173897] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n14:173897] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n14:173897] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n02:167600] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n02:167600] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n02:167600] [11] [a30n07:165009] *** Process received signal *** +[a30n07:165009] Signal: Segmentation fault (11) +[a30n07:165009] Signal code: (3) +[a30n07:165009] Failing at address: 0xb170000000000a0 +[a30n07:165009] [ 0] [0x1000000504d8] +[a30n07:165009] [ 1] [0x3fffcc2e9eb0] +[a30n07:165009] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n07:165009] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n07:165009] [ 4] [b21n14:119484] *** Process received signal *** +[b21n14:119484] Signal: Segmentation fault (11) +[b21n14:119484] Signal code: (3) +[b21n14:119484] Failing at address: 0x14170000000000a0 +[b21n14:119484] [ 0] [0x1000000504d8] +[b21n14:119484] [ 1] [0x3fffcf803640] +[b21n14:119484] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n14:119484] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n11:153417] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n11:153417] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n11:153417] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n11:153417] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n13:168160] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n02:143969] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n02:143969] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n02:143969] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n02:143969] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n02:143969] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n18:138474] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n18:138474] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n18:138474] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n18:138474] [10] [b07n16:166497] *** Process received signal *** +[b07n16:166497] Signal: Segmentation fault (11) +[b07n16:166497] Signal code: (3) +[b07n16:166497] Failing at address: 0xb170000000000a0 +[b07n16:166497] [ 0] [a34n14:150132] *** Process received signal *** +[a34n14:150132] Signal: Segmentation fault (11) +[a34n14:150132] Signal code: (3) +[a34n14:150132] Failing at address: 0x14170000000000a0 +[a34n14:150132] [ 0] [0x1000000504d8] +[a34n14:150132] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n08:129890] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n08:129890] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n05:04519] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n05:04519] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n05:04519] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n05:04519] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n05:04519] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n05:04519] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n05:04519] [13] [a27n03:46689] *** Process received signal *** +[a27n03:46689] Signal: Segmentation fault (11) +[a27n03:46689] Signal code: (3) +[a27n03:46689] Failing at address: 0xb170000000000a0 +[a27n03:46689] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n17:176829] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n15:139748] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n13:117509] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n13:117509] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n13:117509] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n13:117509] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n18:128541] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n18:128541] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n18:128541] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n18:128541] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n11:134750] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n11:134750] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n11:134750] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n11:134750] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n17:171202] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n17:171202] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n07:117360] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n10:11493] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n10:11493] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n10:11493] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n10:11493] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n10:11493] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n10:11493] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n10:11493] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n16:01574] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n16:01574] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n16:01574] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n16:01574] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n17:163820] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n17:163820] [10] [b06n16:165921] *** Process received signal *** +[b06n16:165921] Signal: Segmentation fault (11) +[b06n16:165921] Signal code: (3) +[b06n16:165921] Failing at address: 0x14170000000000a0 +[b06n16:165921] [ 0] [0x1000000504d8] +[b06n16:165921] [ 1] [0x3fffd8ac20e0] +[b06n16:165921] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n16:165921] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n04:52703] [ 4] [b18n11:128958] *** Process received signal *** +[b18n11:128958] Signal: Segmentation fault (11) +[b18n11:128958] Signal code: (3) +[b18n11:128958] Failing at address: 0x14170000000000a0 +[b18n11:128958] [ 0] [0x1000000504d8] +[b18n11:128958] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n13:122031] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n13:122031] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n13:122031] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n13:122031] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n13:122031] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n01:153684] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n01:153684] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n01:153684] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n18:134185] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n18:134185] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n03:130350] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n03:130350] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n03:130350] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n03:130350] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n03:130350] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n14:161621] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n14:161621] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n14:161621] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n14:161621] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n14:161621] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n17:138143] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n17:138143] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n17:138143] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n17:138143] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n16:164554] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n16:164554] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n16:164554] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n16:164554] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n12:140125] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n01:178525] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n01:178525] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n18:69152] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n17:124165] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n17:124165] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n17:124165] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n17:124165] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n17:124165] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n17:124165] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n14:177226] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n14:177226] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n02:125268] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n02:125268] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n02:125268] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n02:125268] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n11:124436] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n11:124436] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n11:124436] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n11:124436] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n06:165306] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n06:165306] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n06:165306] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n06:165306] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n06:165306] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n06:165306] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n06:165306] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n06:165306] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n01:48807] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n14:176068] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n14:176068] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n14:176068] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n14:176068] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n14:176068] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n14:176068] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n14:176068] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n13:10197] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n13:10197] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n13:10197] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n13:10197] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n13:10197] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n13:10197] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n13:10197] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n13:10197] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n13:10197] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n13:10197] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n13:10197] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n07:174563] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n07:174563] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n07:174563] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n07:174563] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n04:138157] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n04:138157] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n12:120403] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n12:120403] *** End of error message *** +[0x1000000504d8] +[b02n13:171108] [ 1] [0x3fffeb18afd0] +[b02n13:171108] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n13:171108] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n13:171108] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n15:123039] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n15:123039] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n15:123039] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n15:123039] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n14:152749] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n14:152749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n14:152749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n14:152749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n14:152749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n14:152749] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n14:152749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n14:152749] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n02:139632] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n02:139632] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n02:139632] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n02:139632] *** End of error message *** +[a35n17:146795] *** Process received signal *** +[a35n17:146795] Signal: Segmentation fault (11) +[a35n17:146795] Signal code: (3) +[a35n17:146795] Failing at address: 0xb170000000000a0 +[a35n17:146795] [ 0] [0x1000000504d8] +[a35n17:146795] [ 1] [0x3ffff847adc0] +[a35n17:146795] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n17:146795] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n17:146795] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n09:51335] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n09:51335] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n09:51335] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n09:51335] [13] [a28n16:171169] *** Process received signal *** +[a28n16:171169] Signal: Segmentation fault (11) +[a28n16:171169] Signal code: (3) +[a28n16:171169] Failing at address: 0x14170000000000a0 +[a28n16:171169] [ 0] [0x1000000504d8] +[a28n16:171169] [ 1] [0x3ffff5bdeda0] +[a28n16:171169] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n16:171169] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n16:171169] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n11:171966] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n11:171966] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n11:171966] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n09:121753] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n09:121753] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n09:121753] [12] [b04n14:08951] *** Process received signal *** +[b04n14:08951] Signal: Segmentation fault (11) +[b04n14:08951] Signal code: (3) +[b04n14:08951] Failing at address: 0x14170000000000a0 +[b04n14:08951] [ 0] [0x1000000504d8] +[b04n14:08951] [ 1] [0x3ffff8beb170] +[b04n14:08951] [ 2] [a31n10:155401] *** Process received signal *** +[a31n10:155401] Signal: Segmentation fault (11) +[a31n10:155401] Signal code: (3) +[a31n10:155401] Failing at address: 0x14170000000000a0 +[a31n10:155401] [ 0] [0x1000000504d8] +[a31n10:155401] [ 1] [0x3ffff4fd7760] +[a31n10:155401] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n10:155401] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n10:176262] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n10:176262] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n10:176262] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n12:122968] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n12:122968] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n15:177057] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n15:177057] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n15:177057] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n15:177057] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n15:177057] [12] [b03n12:167678] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n11:117477] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n11:117477] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n11:117477] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n11:117477] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n11:117477] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n11:117477] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n11:117477] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n07:127359] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n07:127359] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n07:127359] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n07:127359] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n07:127359] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n07:127359] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n07:127359] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n07:127359] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n07:127359] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n01:165128] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n01:165128] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n01:165128] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n01:165128] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n05:148379] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n05:148379] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n05:148379] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n05:148379] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n05:148379] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n03:121811] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n03:121811] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n03:121811] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n03:121811] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n03:121811] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n04:172234] *** End of error message *** +[b21n16:116524] *** Process received signal *** +[b21n16:116524] Signal: Segmentation fault (11) +[b21n16:116524] Signal code: (3) +[b21n16:116524] Failing at address: 0xb170000000000a0 +[b21n16:116524] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n16:124666] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n05:135167] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n05:135167] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n05:135167] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n05:135167] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n05:139438] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n05:139438] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n05:139438] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n05:139438] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n05:139438] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n05:139438] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n05:139438] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n05:139438] [13] [b16n16:127469] *** Process received signal *** +[b16n16:127469] Signal: Segmentation fault (11) +[b16n16:127469] Signal code: (3) +[b16n16:127469] Failing at address: 0xb170000000000a0 +[b16n16:127469] [ 0] [0x1000000504d8] +[b16n16:127469] [ 1] [0x3ffffab3caa0] +[b16n16:127469] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n16:127469] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n09:03929] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n09:03929] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n09:03929] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n09:03929] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n09:03929] *** End of error message *** +[b13n09:143901] *** Process received signal *** +[b13n09:143901] Signal: Segmentation fault (11) +[b13n09:143901] Signal code: (3) +[b13n09:143901] Failing at address: 0xb170000000000a0 +[b15n13:123707] *** Process received signal *** +[b15n13:123707] Signal: Segmentation fault (11) +[b15n13:123707] Signal code: (3) +[b15n13:123707] Failing at address: 0xb170000000000a0 +[b15n13:123707] [ 0] [b07n17:162360] *** Process received signal *** +[b07n17:162360] Signal: Segmentation fault (11) +[b07n17:162360] Signal code: (3) +[b07n17:162360] Failing at address: 0x14170000000000a0 +[b07n17:162360] [ 0] [0x1000000504d8] +[b07n17:162360] [ 1] [b18n06:133957] *** Process received signal *** +[b18n06:133957] Signal: Segmentation fault (11) +[b18n06:133957] Signal code: (3) +[b18n06:133957] Failing at address: 0xb170000000000a0 +[b18n06:133957] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n10:176308] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n10:176308] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n10:176308] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n17:119466] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n17:119466] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n17:119466] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n17:119466] [ 7] [0x3fffc30a1c50] +[a36n18:163111] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n18:163111] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n18:163111] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n18:163111] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n18:163111] [ 6] [ 0] [0x1000000504d8] +[b12n01:145732] [ 1] [0x3fffcbc5cdd0] +[b12n01:145732] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n01:145732] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n01:145732] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n12:145498] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n12:145498] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n12:145498] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n12:145498] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n12:145498] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n12:145498] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n12:145498] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n12:145498] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n12:145498] *** End of error message *** +[a31n12:156916] *** Process received signal *** +[a31n12:156916] Signal: Segmentation fault (11) +[a31n12:156916] Signal code: (3) +[a31n12:156916] Failing at address: 0xb170000000000a0 +[a31n12:156916] [ 0] [0x1000000504d8] +[a31n12:156916] [ 1] [0x3fffd0a07000] +[a31n12:156916] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n12:156916] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n01:141345] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n01:141345] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n01:141345] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n01:141345] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n01:141345] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n01:141345] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n01:141345] [13] [a33n03:147538] *** Process received signal *** +[a33n03:147538] Signal: Segmentation fault (11) +[a33n03:147538] Signal code: (3) +[a33n03:147538] Failing at address: 0x14170000000000a0 +[a33n03:147538] [ 0] [0x1000000504d8] +[a33n03:147538] [ 1] [0x3fffd95783d0] +[a33n03:147538] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n03:147538] [ 3] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n02:139028] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n02:139028] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n17:126528] *** End of error message *** +[b18n14:128869] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n14:128869] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n14:128869] [ 4] [b02n14:169523] *** Process received signal *** +[b02n14:169523] Signal: Segmentation fault (11) +[b02n14:169523] Signal code: (3) +[b02n14:169523] Failing at address: 0x14170000000000a0 +[b02n14:169523] [ 0] [0x1000000504d8] +[b02n14:169523] [ 1] [0x3fffd351a230] +[b02n14:169523] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n14:169523] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n10:10927] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n10:10927] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n10:10927] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n10:10927] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n10:10927] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n04:44760] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n04:44760] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n04:44760] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n04:44760] [10] [b16n14:133633] *** Process received signal *** +[b16n14:133633] Signal: Segmentation fault (11) +[b16n14:133633] Signal code: (3) +[b16n14:133633] Failing at address: 0xb170000000000a0 +[b16n14:133633] [ 0] [0x1000000504d8] +[b16n14:133633] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n14:177514] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n14:177514] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n14:177514] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n14:177514] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n14:177514] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n02:158421] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n02:158421] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n02:158421] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n02:158421] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n02:158421] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n02:158421] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n02:158421] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n02:158421] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n11:148676] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n11:148676] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n11:148676] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n11:148676] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n11:148676] [ 9] [a26n16:53795] *** Process received signal *** +[a26n16:53795] Signal: Segmentation fault (11) +[a26n16:53795] Signal code: (3) +[a26n16:53795] Failing at address: 0x14170000000000a0 +[a26n16:53795] [ 0] [0x1000000504d8] +[a26n16:53795] [ 1] [0x3fffd3d32010] +[a26n16:53795] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n16:53795] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n08:144856] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n08:144856] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n08:144856] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n08:144856] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n08:144856] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n10:122758] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n10:122758] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n10:122758] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n10:122758] [13] [b05n09:106791] *** Process received signal *** +[b05n09:106791] Signal: Segmentation fault (11) +[b05n09:106791] Signal code: (3) +[b05n09:106791] Failing at address: 0x14170000000000a0 +[b05n09:106791] [ 0] [0x1000000504d8] +[b05n09:106791] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n13:165389] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n13:165389] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n14:175713] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n14:175713] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n14:175713] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n01:165167] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n01:165167] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n01:165167] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n07:118165] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n07:118165] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n07:118165] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n07:118165] [14] [b14n16:122493] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n16:122493] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n16:122493] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n16:122493] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n16:122493] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n12:119807] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n12:119807] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n12:119807] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n12:119807] [10] [b10n10:126192] *** Process received signal *** +[b10n10:126192] Signal: Segmentation fault (11) +[b10n10:126192] Signal code: (3) +[b10n10:126192] Failing at address: 0x5170000000000a0 +[b10n10:126192] [ 0] [0x1000000504d8] +[b10n10:126192] [ 1] [0x3fffc8306250] +[b10n10:126192] [ 2] [a28n01:179261] *** Process received signal *** +[a28n01:179261] Signal: Segmentation fault (11) +[a28n01:179261] Signal code: (3) +[a28n01:179261] Failing at address: 0xb170000000000a0 +[a28n01:179261] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n08:118552] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n08:118552] [ 5] [0x1000000504d8] +[b18n05:135124] [ 1] [0x3fffd0668180] +[b18n05:135124] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n05:135124] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n05:135124] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n05:135124] [ 5] [b22n02:119535] *** Process received signal *** +[b22n02:119535] Signal: Segmentation fault (11) +[b22n02:119535] Signal code: (3) +[b22n02:119535] Failing at address: 0x14170000000000a0 +[b22n02:119535] [ 0] [0x1000000504d8] +[b22n02:119535] [ 1] [b06n04:176001] *** Process received signal *** +[b06n04:176001] Signal: Segmentation fault (11) +[b06n04:176001] Signal code: (3) +[b06n04:176001] Failing at address: 0x14170000000000a0 +[b06n04:176001] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n04:117353] *** End of error message *** +[0x1000000504d8] +[b14n12:123563] [ 1] [0x3fffff8eb290] +[b14n12:123563] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n12:123563] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n12:123563] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n06:130542] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n06:130542] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n06:130542] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n06:130542] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n06:130542] [13] [b10n03:168934] *** Process received signal *** +[b10n03:168934] Signal: Segmentation fault (11) +[b10n03:168934] Signal code: (3) +[b10n03:168934] Failing at address: 0xb170000000000a0 +[b10n03:168934] [ 0] [0x1000000504d8] +[b10n03:168934] [ 1] [0x3fffc8c82430] +[b10n03:168934] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n03:168934] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n03:168934] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n03:168934] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n03:168934] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n03:168934] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n03:168934] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n03:168934] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n03:168934] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n03:168934] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n03:168934] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n03:168934] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n03:168934] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n03:168934] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n01:173780] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n01:173780] *** End of error message *** +[0x1000000504d8] +[a29n02:163030] [ 1] [0x3fffd739c9f0] +[a29n02:163030] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n02:163030] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n02:163030] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n02:163030] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n02:163030] [ 6] [a33n01:141817] *** Process received signal *** +[a33n01:141817] Signal: Segmentation fault (11) +[a33n01:141817] Signal code: (3) +[a33n01:141817] Failing at address: 0x14170000000000a0 +[a33n01:141817] [ 0] [0x1000000504d8] +[a33n01:141817] [ 1] [0x3fffcc2cfbe0] +[a33n01:141817] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n01:141817] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n01:141817] [ 4] [0x1000000504d8] +[b13n13:133628] [ 1] [0x3ffff2f8d2b0] +[b13n13:133628] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n13:133628] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n13:133628] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n18:179698] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n18:179698] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n18:179698] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n15:132616] *** End of error message *** +[a34n18:140388] *** Process received signal *** +[a34n18:140388] Signal: Segmentation fault (11) +[a34n18:140388] Signal code: (3) +[a34n18:140388] Failing at address: 0x14170000000000a0 +[a34n18:140388] [ 0] [0x1000000504d8] +[a34n18:140388] [ 1] [0x3ffff5d5ce40] +[a34n18:140388] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n18:140388] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n18:140388] [ 4] [a31n06:167372] *** Process received signal *** +[a31n06:167372] Signal: Segmentation fault (11) +[a31n06:167372] Signal code: (3) +[a31n06:167372] Failing at address: 0x14170000000000a0 +[a31n06:167372] [ 0] [0x1000000504d8] +[a31n06:167372] [ 1] [0x3ffff4de94f0] +[a31n06:167372] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n06:167372] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n06:167372] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n07:153179] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n13:156922] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n13:156922] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n13:156922] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n13:156922] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n13:114396] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n13:114396] *** End of error message *** +[0x1000000504d8] +[b14n01:128003] [ 1] [0x3fffe183b430] +[b14n01:128003] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n01:128003] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n01:128003] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n01:128003] [ 5] [b14n18:131583] *** Process received signal *** +[b14n18:131583] Signal: Segmentation fault (11) +[b14n18:131583] Signal code: (3) +[b14n18:131583] Failing at address: 0xb170000000000a0 +[b14n18:131583] [ 0] [0x1000000504d8] +[b14n18:131583] [ 1] [0x3fffd48f5750] +[b14n18:131583] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n11:130805] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n11:130805] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n11:130805] [ 8] [a32n09:155761] *** Process received signal *** +[a32n09:155761] Signal: Segmentation fault (11) +[a32n09:155761] Signal code: (3) +[a32n09:155761] Failing at address: 0x14170000000000a0 +[a32n09:155761] [ 0] [0x1000000504d8] +[a32n09:155761] [ 1] [0x3ffff225e350] +[a32n09:155761] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n09:155761] [ 3] [0x3fffe768e890] +[b09n18:164801] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n18:164801] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n18:164801] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n18:28218] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n16:166691] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n16:166691] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n16:166691] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n15:120059] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n15:120059] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n15:120059] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n15:120059] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n15:120059] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n15:148165] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n15:148165] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n04:169914] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n04:169914] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n04:169914] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n04:169914] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n13:176365] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n13:176365] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n13:176365] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n13:176365] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n11:120585] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n11:120585] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n11:120585] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n11:120585] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n18:21770] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n18:21770] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n18:21770] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n18:21770] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n13:123053] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n13:123053] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n13:123053] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n13:123053] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n07:133374] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n07:133374] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n07:133374] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n01:124279] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n01:124279] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n01:124279] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n01:124279] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n01:124279] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n15:120131] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n15:120131] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n15:120131] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n03:135039] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n13:177642] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n13:177642] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n13:177642] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n13:177642] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n13:177642] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n13:177642] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n13:177642] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n13:177642] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n13:177642] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n13:177642] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n13:177642] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n04:41673] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n04:41673] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n14:126123] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n14:126123] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n14:126123] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n14:126123] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n12:177121] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n12:177121] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n12:177121] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n12:177121] [10] [b06n13:166927] *** Process received signal *** +[b06n13:166927] Signal: Segmentation fault (11) +[b06n13:166927] Signal code: (3) +[b06n13:166927] Failing at address: 0x14170000000000a0 +[b06n13:166927] [ 0] [0x1000000504d8] +[b06n13:166927] [ 1] [0x3fffdae91750] +[b06n13:166927] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n13:166927] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n13:166927] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n16:162627] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n16:162627] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n16:162627] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n16:162627] [ 9] [b02n10:01256] *** Process received signal *** +[b02n10:01256] Signal: Segmentation fault (11) +[b02n10:01256] Signal code: (3) +[b02n10:01256] Failing at address: 0x14170000000000a0 +[b02n10:01256] [ 0] [0x1000000504d8] +[b02n10:01256] [ 1] [0x3ffff8d5f510] +[b02n10:01256] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n10:01256] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n10:01256] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n10:01256] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n08:08954] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n08:08954] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n08:08954] [ 7] [b21n09:120696] *** Process received signal *** +[b21n09:120696] Signal: Segmentation fault (11) +[b21n09:120696] Signal code: (3) +[b21n09:120696] Failing at address: 0x14170000000000a0 +[b21n09:120696] [ 0] [0x1000000504d8] +[b21n09:120696] [ 1] [0x3ffff170a1b0] +[b21n09:120696] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n07:132091] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n07:132091] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n07:132091] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n06:171502] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n06:171502] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n06:171502] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n06:171502] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n12:115641] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n12:115641] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n12:115641] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n09:149101] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n09:149101] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n09:149101] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n09:149101] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n09:149101] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n08:156370] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n08:156370] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n08:156370] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n08:156370] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n05:149467] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n05:149467] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n05:149467] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n05:149467] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n02:178478] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n02:178478] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n02:178478] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n02:178478] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n15:03735] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n15:03735] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n16:124795] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n16:124795] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n16:124795] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n16:124795] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n04:54380] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n04:54380] *** End of error message *** +[a32n01:158307] *** Process received signal *** +[a32n01:158307] Signal: Segmentation fault (11) +[a32n01:158307] Signal code: (3) +[a32n01:158307] Failing at address: 0x14170000000000a0 +[a32n01:158307] [ 0] [0x1000000504d8] +[a32n01:158307] [ 1] [0x3ffff5b242a0] +[a32n01:158307] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n01:158307] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n01:158307] [ 4] [a36n05:139914] *** Process received signal *** +[a36n05:139914] Signal: Segmentation fault (11) +[a36n05:139914] Signal code: (3) +[a36n05:139914] Failing at address: 0x5170000000000a0 +[a36n05:139914] [ 0] [a34n17:149905] *** Process received signal *** +[a34n17:149905] Signal: Segmentation fault (11) +[a34n17:149905] Signal code: (3) +[a34n17:149905] Failing at address: 0xb170000000000a0 +[a34n17:149905] [ 0] [0x1000000504d8] +[a34n17:149905] [ 1] [0x3fffde569ca0] +[a34n17:149905] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n17:149905] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n12:145267] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n12:145267] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n12:145267] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n12:145267] [10] [a31n07:156491] *** Process received signal *** +[a31n07:156491] Signal: Segmentation fault (11) +[a31n07:156491] Signal code: (3) +[a31n07:156491] Failing at address: 0x14170000000000a0 +[a31n07:156491] [ 0] [b18n03:136669] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n09:173754] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n09:173754] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n09:173754] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n09:173754] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n12:52455] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n12:52455] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n14:139844] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n14:139844] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n10:130581] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n10:130581] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n15:165475] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n02:176052] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n02:176052] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n09:119550] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n09:119550] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n09:119550] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n09:119550] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n10:131276] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n10:131276] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n10:131276] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n10:131276] *** End of error message *** +[ 0] [0x1000000504d8] +[a32n04:153964] [ 1] [0x3fffc318cdd0] +[a32n04:153964] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n04:153964] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n04:153964] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n04:153964] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n04:153964] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n10:161721] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n10:161721] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n10:161721] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n10:161721] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n10:161721] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n10:161721] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n10:161721] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n10:161721] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n14:152041] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n14:152041] *** End of error message *** +[11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n03:135166] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n03:135166] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n17:176415] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n17:176415] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n17:176415] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n17:176415] [ 8] [a28n03:10290] *** Process received signal *** +[a28n03:10290] Signal: Segmentation fault (11) +[a28n03:10290] Signal code: (3) +[a28n03:10290] Failing at address: 0xb170000000000a0 +[a28n03:10290] [ 0] [0x1000000504d8] +[a28n03:10290] [ 1] [0x3fffddec31b0] +[a28n03:10290] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n03:10290] [ 3] [a28n11:01175] *** Process received signal *** +[a28n11:01175] Signal: Segmentation fault (11) +[a28n11:01175] Signal code: (3) +[a28n11:01175] Failing at address: 0x14170000000000a0 +[a28n11:01175] [ 0] [0x1000000504d8] +[a28n11:01175] [ 1] [0x3ffffc941c70] +[a28n11:01175] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n11:01175] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n11:01175] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n16:128440] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n16:128440] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n16:128440] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n01:132064] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n01:132064] *** End of error message *** +[0x1000000504d8] +[b17n13:123518] [ 1] [0x3fffebe70f40] +[b17n13:123518] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n13:123518] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n13:123518] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n06:155089] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n06:155089] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n06:155089] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n03:120672] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n03:120672] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n03:120672] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n03:120672] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n03:120672] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n03:120672] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n03:120672] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n03:120672] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n03:120672] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n03:120672] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n03:120672] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n09:119633] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n09:119633] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n09:119633] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n09:119633] *** End of error message *** +[b03n16:173909] *** Process received signal *** +[b03n16:173909] Signal: Segmentation fault (11) +[b03n16:173909] Signal code: (3) +[b03n16:173909] Failing at address: 0xb170000000000a0 +[b03n16:173909] [ 0] [0x1000000504d8] +[b03n16:173909] [ 1] [0x3fffdba151a0] +[b03n16:173909] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n16:173909] [ 3] [b07n06:177008] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n06:119116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n06:119116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n06:119116] [10] [0x3fffe26529f0] +[b03n13:01301] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n13:01301] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n13:01301] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n13:01301] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n16:171921] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n16:171921] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n16:171921] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n16:171921] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n16:171921] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n16:171921] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n16:171921] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n16:171921] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n15:148584] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n15:177857] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n15:177857] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n08:124727] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n08:124727] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n08:124727] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n08:124727] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n08:124727] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n08:124727] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n16:147288] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n07:156141] *** End of error message *** +[a34n06:148312] *** Process received signal *** +[a34n06:148312] Signal: Segmentation fault (11) +[a34n06:148312] Signal code: (3) +[a34n06:148312] Failing at address: 0x14170000000000a0 +[a34n06:148312] [ 0] [0x1000000504d8] +[a34n06:148312] [ 1] [0x3ffff7ecfc90] +[a34n06:148312] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n06:148312] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n08:138179] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n08:138179] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n08:138179] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n08:138179] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n08:138179] [10] [b08n18:164009] *** Process received signal *** +[b08n18:164009] Signal: Segmentation fault (11) +[b08n18:164009] Signal code: (3) +[b08n18:164009] Failing at address: 0xb170000000000a0 +[b08n18:164009] [ 0] [0x1000000504d8] +[b08n18:164009] [ 1] [0x3fffc7fae410] +[b08n18:164009] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n18:164009] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n18:164009] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n06:127274] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n06:127274] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n06:127274] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n06:127274] [14] [b12n11:137871] *** Process received signal *** +[b12n11:137871] Signal: Segmentation fault (11) +[b12n11:137871] Signal code: (3) +[b12n11:137871] Failing at address: 0xb170000000000a0 +[b12n11:137871] [ 0] [0x1000000504d8] +[b12n11:137871] [ 1] [0x3ffff0ef9b30] +[b12n11:137871] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n11:137871] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n12:12766] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n12:12766] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n12:12766] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n12:12766] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n08:172432] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n08:172432] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n12:123235] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n12:123235] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n12:123235] [11] [b22n17:119581] Signal: Segmentation fault (11) +[b22n17:119581] Signal code: (3) +[b22n17:119581] Failing at address: 0x14170000000000a0 +[b22n17:119581] [ 0] [0x1000000504d8] +[b22n17:119581] [ 1] [0x3fffcb756a40] +[b22n17:119581] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n17:119581] [ 3] [b06n15:161758] *** Process received signal *** +[b06n15:161758] Signal: Segmentation fault (11) +[b06n15:161758] Signal code: (3) +[b06n15:161758] Failing at address: 0x14170000000000a0 +[b06n15:161758] [ 0] [0x1000000504d8] +[b06n15:161758] [ 1] [0x3fffe55b4730] +[b06n15:161758] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n15:161758] [ 3] [b14n08:127444] *** Process received signal *** +[b14n08:127444] Signal: Segmentation fault (11) +[b14n08:127444] Signal code: (3) +[b14n08:127444] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n17:150932] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n17:138617] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n17:138617] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n17:138617] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n11:160346] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n11:160346] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n11:160346] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n11:160346] [12] [0x1000000504d8] +[b11n05:162570] [ 1] [0x3fffc86ad7d0] +[b11n05:162570] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n05:162570] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n05:162570] [ 4] [a31n14:165661] [ 0] [0x1000000504d8] +[a31n14:165661] [ 1] [0x3fffec609910] +[a31n14:165661] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n14:165661] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n14:165661] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n14:165661] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n14:165661] [a35n13:138089] *** Process received signal *** +[a35n13:138089] Signal: Segmentation fault (11) +[a35n13:138089] Signal code: (3) +[a35n13:138089] Failing at address: 0xb170000000000a0 +[a35n13:138089] [ 0] [0x1000000504d8] +[a35n13:138089] [ 1] [a27n02:49109] *** Process received signal *** +[a27n02:49109] Signal: Segmentation fault (11) +[a27n02:49109] Signal code: (3) +[a27n02:49109] Failing at address: 0x14170000000000a0 +[a27n02:49109] [ 0] [0x1000000504d8] +[a27n02:49109] [ 1] [0x3fffed592250] +[a27n02:49109] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n02:49109] [ 3] [a28n04:13083] *** Process received signal *** +[a28n04:13083] Signal: Segmentation fault (11) +[a28n04:13083] Signal code: (3) +[a28n04:13083] Failing at address: 0x14170000000000a0 +[a28n04:13083] [ 0] [0x1000000504d8] +[a28n04:13083] [ 1] [0x3ffffa353c70] +[a28n04:13083] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n04:13083] [ 3] [b14n04:126859] *** Process received signal *** +[b14n04:126859] Signal: Segmentation fault (11) +[b14n04:126859] Signal code: (3) +[b14n04:126859] Failing at address: 0x14170000000000a0 +[b14n04:126859] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n07:120699] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n07:120699] *** End of error message *** +[b02n11:02685] *** Process received signal *** +[b02n11:02685] Signal: Segmentation fault (11) +[b02n11:02685] Signal code: (3) +[b02n11:02685] Failing at address: 0x14170000000000a0 +[b02n11:02685] [ 0] [0x1000000504d8] +[b02n11:02685] [ 1] [0x3ffff9f86d20] +[b02n11:02685] [ 2] [b14n14:122442] *** Process received signal *** +[b14n14:122442] Signal: Segmentation fault (11) +[b14n14:122442] Signal code: (3) +[b14n14:122442] Failing at address: 0xb170000000000a0 +[b14n14:122442] [ 0] [0x1000000504d8] +[b14n14:122442] [ 1] [0x3fffef3ecd90] +[b14n14:122442] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n14:122442] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n14:122442] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n14:122442] [ 5] [a36n15:151253] *** Process received signal *** +[a36n15:151253] Signal: Segmentation fault (11) +[a36n15:151253] Signal code: (3) +[a36n15:151253] Failing at address: 0x14170000000000a0 +[a36n15:151253] [ 0] [b10n02:164493] *** Process received signal *** +[b10n02:164493] Signal: Segmentation fault (11) +[b10n02:164493] Signal code: (3) +[b10n02:164493] Failing at address: 0xb170000000000a0 +[b10n02:164493] [ 0] [0x1000000504d8] +[b10n02:164493] [ 1] [0x3ffff4fd3910] +[b10n02:164493] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n01:164965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n01:164965] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n01:164965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n01:164965] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n08:131381] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n08:131381] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n08:165286] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n08:165286] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n08:165286] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n08:165286] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n09:140257] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n09:140257] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n18:163574] *** End of error message *** +[0x1000000504d8] +[b10n15:164413] [ 1] [0x3fffcc8f4af0] +[b10n15:164413] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n15:164413] [ 3] [a33n11:156153] *** Process received signal *** +[a33n11:156153] Signal: Segmentation fault (11) +[a33n11:156153] Signal code: (3) +[a33n11:156153] Failing at address: 0x5170000000000a0 +[a33n11:156153] [ 0] [0x1000000504d8] +[a33n11:156153] [ 1] [0x3fffeea61ca0] +[a33n11:156153] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n11:156153] [ 3] [b04n13:06928] *** Process received signal *** +[b04n13:06928] Signal: Segmentation fault (11) +[b04n13:06928] Signal code: (3) +[b04n13:06928] Failing at address: 0x14170000000000a0 +[b04n13:06928] [ 0] [0x1000000504d8] +[b04n13:06928] [ 1] [0x3fffd67dcbd0] +[b04n13:06928] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n13:06928] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n03:126188] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n03:126188] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n03:126188] [ 6] [b17n15:137006] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n15:137006] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n15:137006] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n11:120240] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n11:120240] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n11:120240] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n11:120240] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n11:120240] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n11:164998] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n11:164998] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n11:164998] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n11:164998] [ 8] [b23n03:125239] *** Process received signal *** +[b23n03:125239] Signal: Segmentation fault (11) +[b23n03:125239] Signal code: (3) +[b23n03:125239] Failing at address: 0x14170000000000a0 +[b23n03:125239] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n07:172881] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n07:172881] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n07:172881] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n07:172881] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n15:166121] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n15:166121] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n15:166121] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n15:166121] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n15:166121] [ 9] [0x1000000504d8] +[a31n13:155852] [ 1] [0x3ffff8885790] +[a31n13:155852] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n13:155852] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n13:155852] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n13:155852] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n09:137776] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n09:137776] [10] [b18n10:122210] *** Process received signal *** +[b18n10:122210] Signal: Segmentation fault (11) +[b18n10:122210] Signal code: (3) +[b18n10:122210] Failing at address: 0x14170000000000a0 +[b18n10:122210] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n06:154621] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n06:154621] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n06:154621] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n06:154621] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n12:154346] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n12:154346] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n07:138206] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n07:138206] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n07:138206] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n07:138206] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n07:138206] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n07:138206] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n07:138206] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n07:138206] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n05:126783] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n05:126783] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n05:126783] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n05:126783] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n05:126783] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n05:126783] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n05:126783] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n05:126783] [11] [b18n09:123815] *** Process received signal *** +[b18n09:123815] Signal: Segmentation fault (11) +[b18n09:123815] Signal code: (3) +[b18n09:123815] Failing at address: 0xb170000000000a0 +[b18n09:123815] [ 0] [0x1000000504d8] +[b18n09:123815] [ 1] [0x3fffd23e2ea0] +[b18n09:123815] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n09:123815] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n09:123815] [ 4] [a32n12:164381] *** Process received signal *** +[a32n12:164381] Signal: Segmentation fault (11) +[a32n12:164381] Signal code: (3) +[a32n12:164381] Failing at address: 0xb170000000000a0 +[a32n12:164381] [ 0] [0x1000000504d8] +[a32n12:164381] [ 1] [0x3ffff206cd60] +[a32n12:164381] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n11:147714] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n11:147714] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n11:147714] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n11:147714] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n11:147714] [13] [a35n14:136660] *** Process received signal *** +[a35n14:136660] Signal: Segmentation fault (11) +[a35n14:136660] Signal code: (3) +[a35n14:136660] Failing at address: 0xb170000000000a0 +[a35n14:136660] [ 0] [b15n18:08429] *** Process received signal *** +[b15n18:08429] Signal: Segmentation fault (11) +[b15n18:08429] Signal code: (3) +[b15n18:08429] Failing at address: 0xb170000000000a0 +[b15n18:08429] [ 0] [0x1000000504d8] +[b15n18:08429] [ 1] [0x3fffefa4c500] +[b15n18:08429] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n18:08429] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n18:08429] [ 4] [b08n02:162035] *** Process received signal *** +[b08n02:162035] Signal: Segmentation fault (11) +[b08n02:162035] Signal code: (3) +[b08n02:162035] Failing at address: 0x14170000000000a0 +[b08n02:162035] [ 0] [0x1000000504d8] +[b08n02:162035] [ 1] [0x3fffd859d620] +[b08n02:162035] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n02:162035] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n02:162035] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n05:120229] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n05:120229] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n05:120229] [ 8] [b11n09:161233] *** Process received signal *** +[b11n09:161233] Signal: Segmentation fault (11) +[b11n09:161233] Signal code: (3) +[b11n09:161233] Failing at address: 0x14170000000000a0 +[b11n09:161233] [ 0] [0x1000000504d8] +[b11n09:161233] [ 1] [0x3fffd45fa290] +[b11n09:161233] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n09:161233] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n09:161233] [ 4] [a29n12:160187] *** Process received signal *** +[a29n12:160187] Signal: Segmentation fault (11) +[a29n12:160187] Signal code: (3) +[a29n12:160187] Failing at address: 0x14170000000000a0 +[a29n12:160187] [ 0] [0x1000000504d8] +[a29n12:160187] [ 1] [0x3fffd442de90] +[a29n12:160187] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n12:160187] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n05:173848] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n05:173848] [ 6] [a33n08:147821] *** Process received signal *** +[a33n08:147821] Signal: Segmentation fault (11) +[a33n08:147821] Signal code: (3) +[a33n08:147821] Failing at address: 0xb170000000000a0 +[a33n08:147821] [ 0] [0x1000000504d8] +[a33n08:147821] [ 1] [0x3fffd6ee0fa0] +[a33n08:147821] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n08:147821] [ 3] [b03n05:05609] *** Process received signal *** +[b03n05:05609] Signal: Segmentation fault (11) +[b03n05:05609] Signal code: (3) +[b03n05:05609] Failing at address: 0x14170000000000a0 +[b03n05:05609] [ 0] [0x1000000504d8] +[b03n05:05609] [ 1] [0x3fffcc3b2ea0] +[b03n05:05609] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n05:05609] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n05:05609] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n11:152818] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n11:152818] *** End of error message *** +[b10n05:173082] *** Process received signal *** +[b10n05:173082] Signal: Segmentation fault (11) +[b10n05:173082] Signal code: (3) +[b10n05:173082] Failing at address: 0x14170000000000a0 +[b10n05:173082] [ 0] [0x1000000504d8] +[b10n05:173082] [ 1] [0x3fffceb62430] +[b10n05:173082] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n05:173082] [ 3] [b17n18:126908] *** Process received signal *** +[b17n18:126908] Signal: Segmentation fault (11) +[b17n18:126908] Signal code: (3) +[b17n18:126908] Failing at address: 0x14170000000000a0 +[b17n18:126908] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n02:06564] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n02:06564] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n12:132104] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n12:132104] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n12:132104] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n12:132104] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n12:132104] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n12:132104] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n12:132104] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n12:132104] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n12:132104] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n12:132104] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n12:132104] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n12:132104] *** End of error message *** +[b08n01:171787] *** Process received signal *** +[b08n01:171787] Signal: Segmentation fault (11) +[b08n01:171787] Signal code: (3) +[b08n01:171787] Failing at address: 0x14170000000000a0 +[b08n01:171787] [ 0] [0x1000000504d8] +[b08n01:171787] [ 1] [0x3fffd8348e60] +[b08n01:171787] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n01:171787] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n08:163165] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n08:163165] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n08:163165] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n08:163165] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n08:163165] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n08:163165] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n08:163165] [13] [b08n08:164750] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n08:164750] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n08:164750] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n08:164750] [10] [b08n03:175065] *** Process received signal *** +[b08n03:175065] Signal: Segmentation fault (11) +[b08n03:175065] Signal code: (3) +[b08n03:175065] Failing at address: 0x14170000000000a0 +[b08n03:175065] [ 0] [0x1000000504d8] +[b08n03:175065] [ 1] [0x3fffe2118de0] +[b08n03:175065] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n03:175065] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n03:175065] [ 4] [b22n10:118391] *** Process received signal *** +[b22n10:118391] Signal: Segmentation fault (11) +[b22n10:118391] Signal code: (3) +[b22n10:118391] Failing at address: 0x14170000000000a0 +[b22n10:118391] [ 0] [0x1000000504d8] +[b22n10:118391] [ 1] [0x3fffeb2676c0] +[b22n10:118391] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n10:118391] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n10:118391] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n10:118391] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n10:118391] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n10:118391] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n10:118391] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n10:118391] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n10:118391] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n10:118391] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n10:118391] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n10:118391] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n10:118391] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n10:118391] *** End of error message *** +[a31n03:158660] *** Process received signal *** +[a31n03:158660] Signal: Segmentation fault (11) +[a31n03:158660] Signal code: (3) +[a31n03:158660] Failing at address: 0x14170000000000a0 +[a31n03:158660] [ 0] [b12n08:141875] *** Process received signal *** +[b12n08:141875] Signal: Segmentation fault (11) +[b12n08:141875] Signal code: (3) +[b12n08:141875] Failing at address: 0xb170000000000a0 +[b12n08:141875] [ 0] [0x1000000504d8] +[b12n08:141875] [ 1] [0x3ffff74be490] +[b12n08:141875] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n08:141875] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n13:158867] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n13:158867] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n13:158867] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n13:158867] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n13:158867] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n13:158867] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n10:162609] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n10:162609] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n10:162609] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n10:162609] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n13:163193] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n13:163193] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n13:163193] *** End of error message *** +[0x1000000504d8] +[b18n07:125449] [ 1] [0x3fffdd8dc780] +[b18n07:125449] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n07:125449] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n07:125449] [ 4] [a35n02:149229] *** Process received signal *** +[a35n02:149229] Signal: Segmentation fault (11) +[a35n02:149229] Signal code: (3) +[a35n02:149229] Failing at address: 0x14170000000000a0 +[a35n02:149229] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n09:173062] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n09:173062] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n09:173062] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n09:173062] [10] [a31n15:155431] *** Process received signal *** +[a31n15:155431] Signal: Segmentation fault (11) +[a31n15:155431] Signal code: (3) +[a31n15:155431] Failing at address: 0x14170000000000a0 +[a31n15:155431] [ 0] [b08n15:177822] *** Process received signal *** +[b08n15:177822] Signal: Segmentation fault (11) +[b08n15:177822] Signal code: (3) +[b08n15:177822] Failing at address: 0x14170000000000a0 +[b08n15:177822] [ 0] [0x1000000504d8] +[b08n15:177822] [ 1] [0x3fffdd6f5ab0] +[b08n15:177822] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n15:177822] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n15:177822] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n01:138008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n01:138008] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n01:138008] [ 7] [0x1000000504d8] +[b11n06:164017] [ 1] [0x3ffff8f17250] +[b11n06:164017] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n06:164017] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n06:164017] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n06:164017] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n06:164017] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n06:164017] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n06:164017] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n06:164017] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n17:164561] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n17:164561] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n17:164561] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n17:164561] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n18:169363] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n18:169363] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n18:169363] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n18:169363] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n18:169363] *** End of error message *** +[b10n06:162918] *** Process received signal *** +[b10n06:162918] Signal: Segmentation fault (11) +[b10n06:162918] Signal code: (3) +[b10n06:162918] Failing at address: 0xb170000000000a0 +[b10n06:162918] [a28n14:08400] *** Process received signal *** +[a28n14:08400] Signal: Segmentation fault (11) +[a28n14:08400] Signal code: (3) +[a28n14:08400] Failing at address: 0xb170000000000a0 +[a28n14:08400] [ 0] [0x1000000504d8] +[a28n14:08400] [ 1] [0x3fffe8c387e0] +[a28n14:08400] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n14:08400] [ 3] [b13n17:129934] *** Process received signal *** +[b13n17:129934] Signal: Segmentation fault (11) +[b13n17:129934] Signal code: (3) +[b13n17:129934] Failing at address: 0xb170000000000a0 +[b13n17:129934] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n15:122086] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n15:122086] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n15:122086] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n15:122086] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n15:122086] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n15:122086] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n15:122086] [13] [b17n16:127354] *** Process received signal *** +[b17n16:127354] Signal: Segmentation fault (11) +[b17n16:127354] Signal code: (3) +[b17n16:127354] Failing at address: 0xb170000000000a0 +[b17n16:127354] [ 0] [0x1000000504d8] +[b17n16:127354] [ 1] [0x3fffd8dae400] +[b17n16:127354] [ 2] [b06n11:160800] *** Process received signal *** +[b06n11:160800] Signal: Segmentation fault (11) +[b06n11:160800] Signal code: (3) +[b06n11:160800] Failing at address: 0x14170000000000a0 +[b06n11:160800] [ 0] [0x1000000504d8] +[b06n11:160800] [ 1] [0x3fffc2416d30] +[b06n11:160800] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n11:160800] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n11:160800] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n11:160800] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n11:160800] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n18:176812] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n18:176812] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n18:176812] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n18:176812] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n18:176812] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n18:176812] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n18:176812] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n18:176812] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n15:32402] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n15:32402] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n15:32402] [ 6] [ 0] [0x1000000504d8] +[b21n17:118573] [ 1] [0x3fffc7992fc0] +[b21n17:118573] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n17:118573] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n17:118573] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n13:54243] *** End of error message *** +[0x1000000504d8] +[b06n02:163320] [ 1] [0x3fffec7cfe90] +[b06n02:163320] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n02:163320] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n02:163320] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n08:118580] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n08:118580] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n08:118580] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n08:118580] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n13:165260] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n13:165260] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n13:165260] [ 6] [a29n11:162471] [ 0] [0x1000000504d8] +[a29n11:162471] [ 1] [0x3fffd7b838b0] +[a29n11:162471] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n11:162471] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n11:162471] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n02:167600] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n02:167600] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n02:167600] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n02:167600] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n07:165009] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n07:165009] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n07:165009] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n07:165009] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n07:165009] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n07:165009] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n07:165009] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n07:165009] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n07:165009] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n14:119485] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n14:119485] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n14:119485] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n14:119485] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n11:153417] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n11:153417] *** End of error message *** +[b06n18:173977] *** Process received signal *** +[b06n18:173977] Signal: Segmentation fault (11) +[b06n18:173977] Signal code: (3) +[b06n18:173977] Failing at address: 0x14170000000000a0 +[b06n18:173977] [ 0] [b08n09:174690] *** Process received signal *** +[b08n09:174690] Signal: Segmentation fault (11) +[b08n09:174690] Signal code: (3) +[b08n09:174690] Failing at address: 0xb170000000000a0 +[b08n09:174690] [ 0] [0x1000000504d8] +[b08n09:174690] [ 1] [0x3fffd9b9c050] +[b08n09:174690] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n09:174690] [ 3] [b21n15:119252] *** Process received signal *** +[b21n15:119252] Signal: Segmentation fault (11) +[b21n15:119252] Signal code: (3) +[b21n15:119252] Failing at address: 0xb170000000000a0 +[b21n15:119252] [ 0] [0x1000000504d8] +[b21n15:119252] [ 1] [b08n06:161725] [ 0] [0x1000000504d8] +[b08n06:161725] [ 1] [0x3fffd40ae960] +[b08n06:161725] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n06:161725] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n06:161725] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n06:161725] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n14:178324] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n14:178324] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n14:178324] [ 6] [a33n10:140412] *** Process received signal *** +[a33n10:140412] Signal: Segmentation fault (11) +[a33n10:140412] Signal code: (3) +[a33n10:140412] Failing at address: 0x14170000000000a0 +[a33n10:140412] [ 0] [0x1000000504d8] +[a33n10:140412] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n18:118685] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n18:118685] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n18:118685] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n18:118685] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n18:118685] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n18:118685] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n18:118685] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n18:118685] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n18:118685] [13] [a27n07:45307] *** Process received signal *** +[a27n07:45307] Signal: Segmentation fault (11) +[a27n07:45307] Signal code: (3) +[a27n07:45307] Failing at address: 0xb170000000000a0 +[a27n07:45307] [ 0] [0x1000000504d8] +[a27n07:45307] [ 1] [0x3ffffe9fb5c0] +[a27n07:45307] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n07:45307] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n07:45307] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n07:45307] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n07:45307] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n07:45307] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n07:45307] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n07:45307] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n07:45307] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n07:45307] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n07:45307] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n07:45307] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n07:45307] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n07:45307] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n07:03373] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n07:03373] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n07:03373] [ 6] [a29n18:07424] *** Process received signal *** +[a29n18:07424] Signal: Segmentation fault (11) +[a29n18:07424] Signal code: (3) +[a29n18:07424] Failing at address: 0x14170000000000a0 +[a29n18:07424] [ 0] [0x1000000504d8] +[a29n18:07424] [ 1] [0x3fffe3d93a20] +[a29n18:07424] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n18:07424] [ 3] [b06n09:168246] *** Process received signal *** +[b06n09:168246] Signal: Segmentation fault (11) +[b06n09:168246] Signal code: (3) +[b06n09:168246] Failing at address: 0x5170000000000a0 +[b06n09:168246] [ 0] [0x1000000504d8] +[b06n09:168246] [ 1] [0x3ffffe4a70b0] +[b06n09:168246] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n09:168246] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n02:143969] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n02:143969] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n02:143969] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n18:138474] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n18:138474] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n18:138474] [13] [0x1000000504d8] +[a27n03:46689] [ 1] [0x3fffd3e6c280] +[a27n03:46689] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n03:46689] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n03:46689] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n03:46689] [ 5] [0x3ffff06fc0f0] +[a34n14:150132] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n14:150132] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n14:150132] [ 4] [0x1000000504d8] +[b07n16:166497] [ 1] [0x3fffed0f10e0] +[b07n16:166497] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n16:166497] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n16:166497] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n16:166497] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n16:166497] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n05:04519] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n05:04519] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n08:129890] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n08:129890] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n13:117509] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n13:117509] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n11:134747] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n11:134747] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n11:134747] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n11:134747] [12] [b08n17:171200] *** Process received signal *** +[b08n17:171200] Signal: Segmentation fault (11) +[b08n17:171200] Signal code: (3) +[b08n17:171200] Failing at address: 0xb170000000000a0 +[b08n17:171200] [ 0] [0x1000000504d8] +[b08n17:171200] [ 1] [0x3fffea87d1d0] +[b08n17:171200] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n17:171200] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n07:117363] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n07:117363] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n07:117363] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n07:117363] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n16:01574] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n16:01574] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n16:01574] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n16:01574] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n16:01574] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n17:163820] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n17:163820] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n17:163820] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n16:165922] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n16:165922] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n16:165922] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n16:165922] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n04:52703] [0x3fffdd487c80] +[b18n11:128958] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n11:128958] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n11:128958] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n11:128958] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n13:122034] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n13:122034] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n13:122034] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n13:122034] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n13:122034] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n13:122034] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n01:153684] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n01:153684] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n01:153684] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n01:153684] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n18:134185] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n18:134185] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n18:134185] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n18:134185] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n03:130350] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n03:130350] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n03:130350] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n03:130350] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n03:130350] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n03:130350] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n14:161621] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n14:161621] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n14:161621] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n14:161621] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n17:138143] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n17:138143] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n17:138143] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n16:164554] *** End of error message *** +[a02n09:160371] *** Process received signal *** +[a02n09:160371] Signal: Segmentation fault (11) +[a02n09:160371] Signal code: (3) +[a02n09:160371] Failing at address: 0xb170000000000a0 +[a02n09:160371] [ 0] [0x1000000504d8] +[a02n09:160371] [ 1] [0x3fffe726af80] +[a02n09:160371] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n09:160371] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n09:160371] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n17:124168] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n17:124168] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n17:124168] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n17:124168] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n18:69152] [ 8] [b03n14:177223] *** Process received signal *** +[b03n14:177223] Signal: Segmentation fault (11) +[b03n14:177223] Signal code: (3) +[b03n14:177223] Failing at address: 0x5170000000000a0 +[b03n14:177223] [ 0] [0x1000000504d8] +[b03n14:177223] [ 1] [0x3ffffb672d70] +[b03n14:177223] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n14:177223] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n11:124436] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n11:124436] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n14:152746] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n14:152746] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n14:152746] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n06:165306] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n06:165306] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n06:165306] *** End of error message *** +[a27n01:48804] *** Process received signal *** +[a27n01:48804] Signal: Segmentation fault (11) +[a27n01:48804] Signal code: (3) +[a27n01:48804] Failing at address: 0xb170000000000a0 +[a27n01:48804] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n15:123042] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n15:123042] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n15:123042] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n15:123042] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n12:120406] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n12:120406] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n12:120406] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n12:120406] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n12:120406] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n12:120406] [10] [b07n14:176071] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n13:10200] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n13:10200] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n13:10200] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n13:10200] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n13:10200] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n13:10200] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n13:10200] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n13:10200] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n13:10200] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n13:10200] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n13:10200] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n13:10200] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n13:171108] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n13:171108] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n13:171108] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n13:171108] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n07:174563] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n07:174563] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n07:174563] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n10:155398] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n10:155398] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n10:155398] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n10:176262] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n10:176262] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n16:171169] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n16:171169] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n16:171169] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n16:171169] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n16:171169] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n16:171169] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n16:171169] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n16:171169] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n16:171169] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n17:146795] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n17:146795] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n17:146795] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n17:146795] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n09:121753] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n09:121753] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n09:121753] *** End of error message *** +[b04n14:08952] *** Process received signal *** +[b04n14:08952] Signal: Segmentation fault (11) +[b04n14:08952] Signal code: (3) +[b04n14:08952] Failing at address: 0x14170000000000a0 +[b04n14:08952] [ 0] [0x1000000504d8] +[b04n14:08952] [ 1] [0x3fffc6640950] +[b04n14:08952] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n14:08952] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n09:51335] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n09:51335] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n15:177056] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n15:177056] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n15:177056] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n15:177056] [13] [b03n12:167677] *** Process received signal *** +[b03n12:167677] Signal: Segmentation fault (11) +[b03n12:167677] Signal code: (3) +[b03n12:167677] Failing at address: 0x14170000000000a0 +[b03n12:167677] [ 0] [0x1000000504d8] +[b03n12:167677] [ 1] [0x3ffff0812040] +[b03n12:167677] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n12:167677] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n01:165128] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n01:165128] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n11:117477] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n11:117477] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n11:117477] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n11:117477] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n11:117477] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n03:121811] [10] [0x1000000504d8] +[b21n16:116524] [ 1] [0x3fffdb7d32b0] +[b21n16:116524] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n16:116524] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n16:116524] [ 4] [b13n05:135164] *** Process received signal *** +[b13n05:135164] Signal: Segmentation fault (11) +[b13n05:135164] Signal code: (3) +[b13n05:135164] Failing at address: 0xb170000000000a0 +[b13n05:135164] [ 0] [0x1000000504d8] +[b13n05:135164] [ 1] [0x3fffea0e2500] +[b13n05:135164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n05:135164] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n05:148379] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n05:148379] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n05:148379] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n05:148379] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n12:156916] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n12:156916] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n12:156916] [ 6] [a33n03:147535] *** Process received signal *** +[a33n03:147535] Signal: Segmentation fault (11) +[a33n03:147535] Signal code: (3) +[a33n03:147535] Failing at address: 0x5170000000000a0 +[a33n03:147535] [ 0] [0x1000000504d8] +[a33n03:147535] [ 1] [0x3fffed090ef0] +[a33n03:147535] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n03:147535] [ 3] [b13n09:143901] [ 0] [0x1000000504d8] +[b13n09:143901] [ 1] [0x3fffdc479410] +[b13n09:143901] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n09:143901] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n09:143901] [ 4] [b07n17:162361] *** Process received signal *** +[b07n17:162361] Signal: Segmentation fault (11) +[b07n17:162361] Signal code: (3) +[b07n17:162361] Failing at address: 0x14170000000000a0 +[b07n17:162361] [ 0] [0x1000000504d8] +[b07n17:162361] [ 1] [0x3ffffea6c080] +[b07n17:162361] [ 2] [0x1000000504d8] +[b18n06:133957] [ 1] [0x3fffebce3d30] +[b18n06:133957] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n06:133957] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n06:133957] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n01:145732] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n01:145732] [ 6] [0x1000000504d8] +[b15n13:123707] [ 1] [0x3fffe7864920] +[b15n13:123707] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n13:123707] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n13:123707] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n01:141345] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n01:141345] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n16:127468] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n16:127468] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n16:127468] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n16:127468] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n16:127468] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n16:127468] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n16:127468] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n16:127468] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n16:127468] [12] [a28n09:03930] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n10:176308] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n10:176308] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n10:176308] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n17:119469] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n17:119469] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n18:163111] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n18:163111] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n18:163111] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n18:163111] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n18:163111] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n18:163111] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n18:163111] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n18:163111] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n18:163111] *** End of error message *** +[a29n13:165390] *** Process received signal *** +[a29n13:165390] Signal: Segmentation fault (11) +[a29n13:165390] Signal code: (3) +[a29n13:165390] Failing at address: 0x14170000000000a0 +[a29n13:165390] [ 0] [0x1000000504d8] +[a29n13:165390] [ 1] [0x3ffffa51bcd0] +[a29n13:165390] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n13:165390] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n13:165390] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n13:165390] [ 5] [0x1000000504d8] +[b23n07:118168] [ 1] [0x3ffff2cd9760] +[b23n07:118168] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n07:118168] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n07:118168] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n07:118168] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n02:158421] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n02:158421] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n02:158421] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n02:158421] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n14:128872] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n14:128872] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n14:128872] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n14:128872] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n14:128872] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n08:144856] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n08:144856] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n08:144856] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n08:144856] [13] [b05n09:106788] *** Process received signal *** +[b05n09:106788] Signal: Segmentation fault (11) +[b05n09:106788] Signal code: (3) +[b05n09:106788] Failing at address: 0xb170000000000a0 +[b05n09:106788] [ 0] [0x1000000504d8] +[b05n09:106788] [ 1] [0x3fffceb6b100] +[b05n09:106788] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n09:106788] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n09:106788] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n01:165164] *** End of error message *** +[0x3fffc9612980] +[b16n14:133633] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n14:133633] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n14:133633] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n14:133633] [ 5] [b02n14:169524] *** Process received signal *** +[b02n14:169524] Signal: Segmentation fault (11) +[b02n14:169524] Signal code: (3) +[b02n14:169524] Failing at address: 0x14170000000000a0 +[b02n14:169524] [ 0] [0x1000000504d8] +[b02n14:169524] [ 1] [0x3ffffaa7ae60] +[b02n14:169524] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n14:169524] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n10:10924] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n10:10924] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n10:10924] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n10:10924] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n10:10924] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n10:126195] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n17:126525] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n17:126525] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n04:44760] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n04:44760] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n04:44760] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n12:119807] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n12:119807] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n12:119807] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n12:119807] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n11:148676] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n11:148676] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n11:148676] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n11:148676] [13] [ 0] [0x1000000504d8] +[a28n01:179261] [ 1] [0x3fffdf7bc430] +[a28n01:179261] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n01:179261] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n01:179261] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n10:122758] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n10:122758] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n14:175713] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n14:175713] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n14:175713] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n14:175713] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n14:175713] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n16:53795] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n16:53795] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n16:53795] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n08:118552] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n08:118552] [ 7] [0x3fffef499670] +[b22n02:119535] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n02:119535] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n02:119535] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n18:179701] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n18:179701] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n18:179701] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n04:117356] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n04:117356] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n04:117356] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n04:117356] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n04:117356] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n01:141817] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n01:141817] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n01:141817] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n01:141817] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n01:141817] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n01:141817] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n12:123563] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n12:123563] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n12:123563] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n12:123563] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n12:123563] [ 9] [a36n07:153176] *** Process received signal *** +[a36n07:153176] Signal: Segmentation fault (11) +[a36n07:153176] Signal code: (3) +[a36n07:153176] Failing at address: 0xb170000000000a0 +[a36n07:153176] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n13:133628] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n13:133628] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n13:133628] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n13:133628] [ 8] [0x1000000504d8] +[b06n04:176001] [ 1] [0x3ffff9e7dd60] +[b06n04:176001] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n04:176001] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n04:176001] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n02:163030] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n02:163030] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n02:163030] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n02:163030] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n02:163030] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n02:163030] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n03:168931] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n03:168931] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n03:168931] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n03:168931] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n03:168931] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n13:156922] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n13:156922] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n13:156922] [11] [b23n13:114399] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n18:131583] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n18:131583] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n18:131583] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n11:130805] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n11:130805] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n11:130805] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n11:130805] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n11:130805] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n18:28218] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n18:28218] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n18:28218] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n18:28218] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n15:120062] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n15:120062] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n15:120062] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n15:120062] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n15:120062] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n15:120062] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n15:120062] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n15:120062] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n15:120062] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n15:120062] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n15:120062] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n15:120062] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n04:169914] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n04:169914] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n04:169914] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n13:123053] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n13:123053] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n13:176365] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n11:120585] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n11:120585] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n15:120131] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n15:120131] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n12:177121] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n12:177121] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n12:177121] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n12:177121] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n13:166927] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n13:166927] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n13:166927] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n13:166927] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n16:162627] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n16:162627] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n16:162627] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n16:162627] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n10:01256] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n10:01256] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n10:01256] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n10:01256] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n08:08954] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n08:08954] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n08:08954] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n07:132091] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n07:132091] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n07:132091] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n12:115641] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n12:115641] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n12:115641] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n12:115641] [12] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n09:120696] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n09:120696] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n09:120696] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n09:120696] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n09:120696] [ 7] [a30n09:152063] *** Process received signal *** +[a30n09:152063] Signal: Segmentation fault (11) +[a30n09:152063] Signal code: (3) +[a30n09:152063] Failing at address: 0x14170000000000a0 +[a30n09:152063] [ 0] [0x1000000504d8] +[a30n09:152063] [ 1] [0x3fffd0aa6020] +[a30n09:152063] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n09:152063] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n09:152063] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n06:171502] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n06:171502] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n02:178478] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n02:178478] *** End of error message *** +[0x1000000504d8] +[a36n05:139914] [ 1] [0x3ffff53557c0] +[a36n05:139914] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n05:139914] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n05:139914] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n05:139914] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n05:139914] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n05:139914] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n05:139914] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n05:139914] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n05:139914] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n05:139914] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n05:139914] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n05:139914] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n05:139914] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n05:139914] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n09:173754] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n10:130584] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n10:130584] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n10:130584] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n10:130584] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n10:130584] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n01:128003] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n01:128003] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n01:128003] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n01:128003] [ 9] [b05n02:176049] *** Process received signal *** +[b05n02:176049] Signal: Segmentation fault (11) +[b05n02:176049] Signal code: (3) +[b05n02:176049] Failing at address: 0xb170000000000a0 +[b05n02:176049] [ 0] [b22n01:122016] *** Process received signal *** +[b22n01:122016] Signal: Segmentation fault (11) +[b22n01:122016] Signal code: (3) +[b22n01:122016] Failing at address: 0x14170000000000a0 +[b22n01:122016] [ 0] [0x1000000504d8] +[b22n01:122016] [ 1] [0x3fffd9434a20] +[b22n01:122016] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n01:122016] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n01:122016] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n03:10290] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n03:10290] [ 5] [0x3fffe78324b0] +[b14n09:119547] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n09:119547] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n09:119547] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n09:119547] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n11:01175] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n11:01175] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n11:01175] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n11:01175] [ 8] [b05n15:176426] *** Process received signal *** +[b05n15:176426] Signal: Segmentation fault (11) +[b05n15:176426] Signal code: (3) +[b05n15:176426] Failing at address: 0x5170000000000a0 +[b05n15:176426] [ 0] [0x1000000504d8] +[b05n15:176426] [ 1] [0x3fffc8e0baa0] +[b05n15:176426] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n15:176426] [ 3] [b03n16:173910] *** Process received signal *** +[b03n16:173910] Signal: Segmentation fault (11) +[b03n16:173910] Signal code: (3) +[b03n16:173910] Failing at address: 0xb170000000000a0 +[b03n16:173910] [ 0] [0x1000000504d8] +[b03n16:173910] [ 1] [0x3ffffd279b80] +[b03n16:173910] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n16:173910] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n13:01302] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n13:01302] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n16:171917] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n16:171917] *** End of error message *** +[b04n15:177862] *** Process received signal *** +[b04n15:177862] Signal: Segmentation fault (11) +[b04n15:177862] Signal code: (3) +[b04n15:177862] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n08:124727] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n08:124727] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n08:124727] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n08:124727] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n08:124727] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n08:124727] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n18:164009] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n18:164009] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n18:164801] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n18:164801] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n11:137871] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n11:137871] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n11:137871] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n11:137871] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n11:137871] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n01:173783] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n01:173783] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n01:173783] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n01:173783] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n01:173783] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n01:173783] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n01:173783] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n01:173783] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n15:148165] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n15:148165] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n15:148165] [13] [b06n15:161755] *** Process received signal *** +[b06n15:161755] Signal: Segmentation fault (11) +[b06n15:161755] Signal code: (3) +[b06n15:161755] Failing at address: 0x5170000000000a0 +[b06n15:161755] [ 0] [0x1000000504d8] +[b06n15:161755] [ 1] [0x3ffffadfea00] +[b06n15:161755] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n15:161755] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n08:127447] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n08:127447] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n08:127447] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n08:127447] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n17:150932] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n02:49109] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n02:49109] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n02:49109] [ 6] [b13n06:133803] *** Process received signal *** +[b13n06:133803] Signal: Segmentation fault (11) +[b13n06:133803] Signal code: (3) +[b13n06:133803] Failing at address: 0xb170000000000a0 +[b13n06:133803] [ 0] [0x1000000504d8] +[b13n06:133803] [ 1] [0x3fffc5f6c030] +[b13n06:133803] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n06:133803] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n06:133803] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n06:133803] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n06:133803] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n06:133803] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n06:133803] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n06:133803] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n06:133803] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n06:133803] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n06:133803] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n06:133803] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n06:133803] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n06:133803] *** End of error message *** +[b02n11:02686] *** Process received signal *** +[b02n11:02686] Signal: Segmentation fault (11) +[b02n11:02686] Signal code: (3) +[b02n11:02686] Failing at address: 0x14170000000000a0 +[b02n11:02686] [ 0] [0x1000000504d8] +[b02n11:02686] [ 1] [0x3fffef0c3e80] +[b02n11:02686] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n11:02686] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n05:122273] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n05:122273] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n05:122273] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n05:122273] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n05:122273] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n05:122273] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n05:122273] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n05:122273] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n05:122273] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n07:168965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n07:168965] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n07:168965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n07:168965] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n06:155089] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n06:155089] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n06:155089] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n06:155089] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n06:155089] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n11:156153] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n11:156153] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n11:156153] [ 6] [b04n13:06927] *** Process received signal *** +[b04n13:06927] Signal: Segmentation fault (11) +[b04n13:06927] Signal code: (3) +[b04n13:06927] Failing at address: 0xb170000000000a0 +[b04n13:06927] [ 0] [0x1000000504d8] +[b04n13:06927] [ 1] [0x3ffff3439a60] +[b04n13:06927] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n13:06927] [ 3] [a26n03:68833] *** Process received signal *** +[a26n03:68833] Signal: Segmentation fault (11) +[a26n03:68833] Signal code: (3) +[a26n03:68833] Failing at address: 0x14170000000000a0 +[a26n03:68833] [ 0] [0x1000000504d8] +[a26n03:68833] [ 1] [0x3fffebedafe0] +[a26n03:68833] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n03:68833] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n03:68833] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n03:68833] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n14:122442] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n14:122442] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n14:122442] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n14:122442] [ 9] [0x1000000504d8] +[a36n15:151253] [ 1] [0x3fffde168420] +[a36n15:151253] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n15:151253] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n15:151253] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n15:151253] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n10:161718] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n10:161718] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n10:161718] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n10:161718] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n10:161718] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n06:167372] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n06:167372] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n06:167372] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n06:167372] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n06:167372] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n05:149467] *** End of error message *** +[a33n09:147792] *** Process received signal *** +[a33n09:147792] Signal: Segmentation fault (11) +[a33n09:147792] Signal code: (3) +[a33n09:147792] Failing at address: 0xb170000000000a0 +[a33n09:147792] [ 0] [0x1000000504d8] +[a33n09:147792] [ 1] [0x3fffe21b1760] +[a33n09:147792] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n09:147792] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n09:147792] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n04:13082] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n04:13082] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n04:13082] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n04:13082] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n04:13082] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n04:13082] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n04:13082] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n07:133374] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n07:133374] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n07:133374] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n07:133374] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n10:11490] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n10:11490] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n10:11490] [11] [b01n08:172428] *** Process received signal *** +[b01n08:172428] Signal: Segmentation fault (11) +[b01n08:172428] Signal code: (3) +[b01n08:172428] Failing at address: 0x14170000000000a0 +[b01n08:172428] [ 0] [0x1000000504d8] +[b01n08:172428] [ 1] [0x3fffec763050] +[b01n08:172428] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n08:172428] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n16:124796] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n16:124796] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n16:124796] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n16:124796] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n16:124796] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n16:124796] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n05:162570] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n05:162570] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n12:145267] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n12:145267] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n12:145267] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n12:145267] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n08:121435] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n08:121435] *** End of error message *** +[0x1000000504d8] +[b23n03:125239] [ 1] [0x3fffd7c05a60] +[b23n03:125239] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n03:125239] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n03:125239] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n06:130545] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n06:130545] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n18:140385] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n18:140385] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n18:140385] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n18:140385] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n18:140385] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n18:140385] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n18:140385] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n16:128440] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n16:128440] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n16:128440] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n16:128440] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n05:120229] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n05:120229] [10] [0x3fffda2e2930] +[a29n12:160188] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n12:160188] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n12:160188] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n12:160188] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n17:119581] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n17:119581] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n05:135124] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n05:135124] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n05:135124] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n05:135124] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n05:135124] [10] [b06n03:164496] *** Process received signal *** +[b06n03:164496] Signal: Segmentation fault (11) +[b06n03:164496] Signal code: (3) +[b06n03:164496] Failing at address: 0x14170000000000a0 +[b06n03:164496] [ 0] [0x1000000504d8] +[b06n03:164496] [ 1] [0x3ffff7d82db0] +[b06n03:164496] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n03:164496] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n03:164496] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n15:166121] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n15:166121] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n15:166121] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n15:166121] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n08:138179] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n08:138179] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n08:138179] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n08:156370] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n08:156370] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n06:127274] *** End of error message *** +[b05n04:06832] *** Process received signal *** +[b05n04:06832] Signal: Segmentation fault (11) +[b05n04:06832] Signal code: (3) +[b05n04:06832] Failing at address: 0x14170000000000a0 +[b05n04:06832] [ 0] [0x1000000504d8] +[b05n04:06832] [ 1] [0x3fffc0f99f20] +[b05n04:06832] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n03:126188] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n03:126188] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n03:126188] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n03:126188] [10] [0x1000000504d8] +[b08n04:172231] [ 1] [0x3fffc35cb400] +[b08n04:172231] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n04:172231] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n04:172231] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n04:172231] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n04:172231] [ 6] [b22n14:116415] *** Process received signal *** +[b22n14:116415] Signal: Segmentation fault (11) +[b22n14:116415] Signal code: (3) +[b22n14:116415] Failing at address: 0x14170000000000a0 +[b22n14:116415] [ 0] [b18n16:124962] *** Process received signal *** +[b18n16:124962] Signal: Segmentation fault (11) +[b18n16:124962] Signal code: (3) +[b18n16:124962] Failing at address: 0x14170000000000a0 +[b18n16:124962] [ 0] [0x1000000504d8] +[b18n16:124962] [ 1] [0x3fffd1ddde40] +[b18n16:124962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n16:124962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n16:124962] [ 4] [a02n06:38120] *** Process received signal *** +[a02n06:38120] Signal: Segmentation fault (11) +[a02n06:38120] Signal code: (3) +[a02n06:38120] Failing at address: 0x14170000000000a0 +[a02n06:38120] [ 0] [0x1000000504d8] +[a02n06:38120] [ 1] [0x3ffff4d3c430] +[a02n06:38120] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n06:38120] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n06:38120] [ 4] [a32n03:151884] *** Process received signal *** +[a32n03:151884] Signal: Segmentation fault (11) +[a32n03:151884] Signal code: (3) +[a32n03:151884] Failing at address: 0x5170000000000a0 +[a32n03:151884] [ 0] [0x1000000504d8] +[a32n03:151884] [ 1] [0x3ffffc411710] +[a32n03:151884] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n03:151884] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n03:151884] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n09:161230] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n09:161230] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n09:161230] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n09:161230] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n09:161230] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n09:161230] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n09:161230] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n09:161230] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n09:161230] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n09:149101] [10] [0x1000000504d8] +[a31n07:156491] [ 1] [0x3fffeebef960] +[a31n07:156491] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n07:156491] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n07:156491] [ 4] [0x3ffffb7a3860] +[a35n13:138089] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n13:138089] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n13:138089] [ 4] [a26n08:76225] *** Process received signal *** +[a26n08:76225] Signal: Segmentation fault (11) +[a26n08:76225] Signal code: (3) +[a26n08:76225] Failing at address: 0x5170000000000a0 +[a26n08:76225] [ 0] [0x1000000504d8] +[a26n08:76225] [ 1] [0x3ffffd63d720] +[a26n08:76225] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n08:76225] [ 3] [0x1000000504d8] +[b14n04:126859] [ 1] [0x3fffce0eaa20] +[b14n04:126859] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n04:126859] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n04:126859] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n09:155758] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n09:155758] *** End of error message *** +[ 0] [0x1000000504d8] +[a34n17:149904] [ 1] [0x3fffc12f6cc0] +[a34n17:149904] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n17:149904] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n17:149904] [ 4] [b17n14:126120] *** Process received signal *** +[b17n14:126120] Signal: Segmentation fault (11) +[b17n14:126120] Signal code: (3) +[b17n14:126120] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n12:148380] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n12:148380] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n12:148380] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n12:148380] [ 8] [b10n11:160351] [ 0] [0x1000000504d8] +[b10n11:160351] [ 1] [0x3fffffb49430] +[b10n11:160351] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n11:160351] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n11:160351] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n11:160351] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n03:135166] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n17:176415] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n17:176415] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n17:176415] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n17:176415] [12] [b20n13:131758] *** Process received signal *** +[b20n13:131758] Signal: Segmentation fault (11) +[b20n13:131758] Signal code: (3) +[b20n13:131758] Failing at address: 0xb170000000000a0 +[b20n13:131758] [ 0] [0x1000000504d8] +[b20n13:131758] [ 1] [0x3fffddba5a50] +[b20n13:131758] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n13:131758] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n02:164490] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n02:164490] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n06:148312] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n06:148312] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n06:148312] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n06:148312] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n06:148312] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n09:137776] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n09:137776] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n09:137776] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n09:137776] [14] [a28n17:168164] *** Process received signal *** +[a28n17:168164] Signal: Segmentation fault (11) +[a28n17:168164] Signal code: (3) +[a28n17:168164] Failing at address: 0x14170000000000a0 +[a28n17:168164] [ 0] [0x1000000504d8] +[a28n17:168164] [ 1] [b14n05:126780] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n05:126780] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n05:126780] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n05:126780] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n01:158307] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n01:158307] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n01:158307] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n01:158307] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n01:158307] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n01:158307] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n01:158307] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n01:158307] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n01:158307] [13] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n14:165661] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n14:165661] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n14:165661] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n14:165661] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n14:165661] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n14:165661] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n07:138206] *** End of error message *** +[a26n09:62480] *** Process received signal *** +[a26n09:62480] Signal: Segmentation fault (11) +[a26n09:62480] Signal code: (3) +[a26n09:62480] Failing at address: 0x14170000000000a0 +[a26n09:62480] [ 0] [0x1000000504d8] +[a26n09:62480] [ 1] [0x3ffff4635020] +[a26n09:62480] [ 2] [b12n14:139845] *** Process received signal *** +[b12n14:139845] Signal: Segmentation fault (11) +[b12n14:139845] Signal code: (3) +[b12n14:139845] Failing at address: 0x5170000000000a0 +[b12n14:139845] [ 0] [b18n03:136666] *** Process received signal *** +[b18n03:136666] Signal: Segmentation fault (11) +[b18n03:136666] Signal code: (3) +[b18n03:136666] Failing at address: 0xb170000000000a0 +[b18n03:136666] [ 0] [b06n14:173894] *** Process received signal *** +[b06n14:173894] Signal: Segmentation fault (11) +[b06n14:173894] Signal code: (3) +[b06n14:173894] Failing at address: 0x5170000000000a0 +[b06n14:173894] [ 0] [b16n13:122187] *** Process received signal *** +[b16n13:122187] Signal: Segmentation fault (11) +[b16n13:122187] Signal code: (3) +[b16n13:122187] Failing at address: 0xb170000000000a0 +[b16n13:122187] [ 0] [0x1000000504d8] +[b16n13:122187] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n16:147288] [ 6] [0x3fffee32bcc0] +[a34n01:164962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n01:164962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n01:164962] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n01:164962] [ 5] [0x1000000504d8] +[a35n14:136660] [ 1] [0x3fffde7fae10] +[a35n14:136660] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n14:136660] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n14:136660] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n14:136660] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n13:123518] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n13:123518] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n13:123518] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n13:123518] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n13:123518] [ 9] [a02n10:135855] *** Process received signal *** +[a02n10:135855] Signal: Segmentation fault (11) +[a02n10:135855] Signal code: (3) +[a02n10:135855] Failing at address: 0x14170000000000a0 +[a02n10:135855] [ 0] [0x1000000504d8] +[a02n10:135855] [ 1] [0x3fffc1cff2f0] +[a02n10:135855] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n10:135855] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n10:135855] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n17:138614] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n17:138614] *** End of error message *** +[b03n05:05608] *** Process received signal *** +[b03n05:05608] Signal: Segmentation fault (11) +[b03n05:05608] Signal code: (3) +[b03n05:05608] Failing at address: 0x14170000000000a0 +[b03n05:05608] [ 0] [0x1000000504d8] +[b03n05:05608] [ 1] [0x3ffff599cf20] +[b03n05:05608] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n05:05608] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n05:05608] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n18:08432] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n18:08432] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n18:08432] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n18:08432] [ 8] [b03n02:06561] *** Process received signal *** +[b01n13:177640] *** Process received signal *** +[b01n13:177640] Signal: Segmentation fault (11) +[b01n13:177640] Signal code: (3) +[b01n13:177640] Failing at address: 0x14170000000000a0 +[b01n13:177640] [ 0] [0x1000000504d8] +[b01n13:177640] [ 1] [0x3ffff5a1f390] +[b01n13:177640] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n13:177640] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n12:123235] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n12:123235] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n12:123235] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n10:118388] Signal: Segmentation fault (11) +[b22n10:118388] Signal code: (3) +[b22n10:118388] Failing at address: 0xb170000000000a0 +[b22n10:118388] [ 0] [0x1000000504d8] +[b22n10:118388] [ 1] [0x3fffd3081a20] +[b22n10:118388] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n10:118388] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n08:141875] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n08:141875] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n08:141875] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n08:141875] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n09:173062] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n09:173062] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n09:173062] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n06:164016] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n06:164016] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n06:164016] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n06:164016] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n06:164016] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n06:164016] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n14:08400] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n14:08400] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n14:08400] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n11:160800] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n11:160800] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n11:160800] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n11:160800] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n11:160800] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n11:160800] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n11:160800] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n11:160800] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n11:160800] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n15:32402] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n15:32402] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n15:32402] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n18:176810] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n18:176810] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n18:176810] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n18:176810] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n17:118573] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n17:118573] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n02:163320] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n02:163320] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n02:163320] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n02:163320] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n02:163320] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n08:118580] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n08:118580] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n08:118580] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n11:162474] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n11:162474] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n11:162474] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n11:162474] *** End of error message *** +[a33n02:167597] *** Process received signal *** +[a33n02:167597] Signal: Segmentation fault (11) +[a33n02:167597] Signal code: (3) +[a33n02:167597] Failing at address: 0x5170000000000a0 +[0x1000000504d8] +[b06n18:173977] [ 1] [0x3fffc42a3360] +[b06n18:173977] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n18:173977] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n18:173977] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n18:173977] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n18:173977] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n18:173977] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n18:173977] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n18:173977] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n14:119484] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n14:119484] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n14:119484] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n14:119484] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n14:119484] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n14:119484] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n14:119484] [10] [b21n15:119251] *** Process received signal *** +[b21n15:119251] Signal: Segmentation fault (11) +[b21n15:119251] Signal code: (3) +[b21n15:119251] Failing at address: 0xb170000000000a0 +[b21n15:119251] [ 0] [0x1000000504d8] +[b21n15:119251] [ 1] [0x3fffeb938550] +[b21n15:119251] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n15:119251] [ 3] [ 0] [0x1000000504d8] +[b10n06:162918] [ 1] [0x3fffe07c1d90] +[b10n06:162918] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n06:162918] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n06:162918] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n11:147714] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n11:147714] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n13:155852] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n13:155852] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n13:155852] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n13:155852] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n13:155852] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n13:155852] [11] [b18n04:136998] *** Process received signal *** +[b18n04:136998] Signal: Segmentation fault (11) +[b18n04:136998] Signal code: (3) +[b18n04:136998] Failing at address: 0xb170000000000a0 +[b18n04:136998] [ 0] [0x1000000504d8] +[b18n04:136998] [ 1] [0x3ffff13a3d20] +[b18n04:136998] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n04:136998] [ 3] [b12n06:135850] *** Process received signal *** +[b12n06:135850] Signal: Segmentation fault (11) +[b12n06:135850] Signal code: (3) +[b12n06:135850] Failing at address: 0x14170000000000a0 +[b12n06:135850] [ 0] [0x1000000504d8] +[b12n06:135850] [ 1] [0x3fffc6800aa0] +[b12n06:135850] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n06:135850] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n08:165286] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n17:164564] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n17:164564] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n17:164564] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n17:164564] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n08:163165] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n08:163165] *** End of error message *** +[a27n14:21318] *** Process received signal *** +[a27n14:21318] Signal: Segmentation fault (11) +[a27n14:21318] Signal code: (3) +[a27n14:21318] Failing at address: 0xb170000000000a0 +[a27n14:21318] [ 0] [0x1000000504d8] +[a27n14:21318] [ 1] [0x3fffcc488a90] +[a27n14:21318] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n14:21318] [ 3] [0x1000000504d8] +[b13n17:129934] [ 1] [0x3fffc343e560] +[b13n17:129934] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n17:129934] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n17:129934] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n17:129934] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n17:129934] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n17:129934] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n17:129934] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n15:122086] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n15:122086] *** End of error message *** +[b18n08:131378] *** Process received signal *** +[b18n08:131378] Signal: Segmentation fault (11) +[b18n08:131378] Signal code: (3) +[b18n08:131378] Failing at address: 0x14170000000000a0 +[b18n08:131378] [ 0] [b03n18:177222] *** Process received signal *** +[b03n18:177222] Signal: Segmentation fault (11) +[b03n18:177222] Signal code: (3) +[b03n18:177222] Failing at address: 0xb170000000000a0 +[b03n18:177222] [ 0] [0x1000000504d8] +[b03n18:177222] [ 1] [0x3fffe583a660] +[b03n18:177222] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n18:177222] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n07:172881] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n07:172881] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n07:172881] [13] [0x1000000504d8] +[a35n02:149229] [ 1] [0x3fffc6b19c20] +[a35n02:149229] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n02:149229] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n02:149229] [ 4] [b05n03:177924] *** Process received signal *** +[b05n03:177924] Signal: Segmentation fault (11) +[b05n03:177924] Signal code: (3) +[b05n03:177924] Failing at address: 0x14170000000000a0 +[b05n03:177924] [ 0] [0x1000000504d8] +[b05n03:177924] [ 1] [0x3fffd54599d0] +[b05n03:177924] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n03:177924] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n03:177924] [ 4] [b12n05:138196] *** Process received signal *** +[b12n05:138196] Signal: Segmentation fault (11) +[b12n05:138196] Signal code: (3) +[b12n05:138196] Failing at address: 0xb170000000000a0 +[b12n05:138196] [ 0] [0x1000000504d8] +[b12n05:138196] [ 1] [0x3fffe719f680] +[b12n05:138196] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n05:138196] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n15:137006] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n15:137006] *** End of error message *** +[a27n07:45310] *** Process received signal *** +[a27n07:45310] Signal: Segmentation fault (11) +[a27n07:45310] Signal code: (3) +[a27n07:45310] Failing at address: 0x14170000000000a0 +[a27n07:45310] [ 0] [0x1000000504d8] +[a27n07:45310] [ 1] [0x3fffe3121b30] +[a27n07:45310] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n07:45310] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n07:45310] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n07:45310] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n07:45310] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n07:45310] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n07:45310] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n07:45310] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n07:45310] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n07:45310] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n07:45310] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n07:45310] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n07:45310] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n07:45310] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n07:03373] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n07:03373] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n07:03373] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n07:03373] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n02:143969] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n02:143969] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n02:143969] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n18:138474] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n18:138474] *** End of error message *** +[a29n18:07425] *** Process received signal *** +[a29n18:07425] Signal: Segmentation fault (11) +[a29n18:07425] Signal code: (3) +[a29n18:07425] Failing at address: 0x14170000000000a0 +[a29n18:07425] [ 0] [0x1000000504d8] +[a29n18:07425] [ 1] [0x3fffdc04b280] +[a29n18:07425] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n18:07425] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n18:07425] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n18:07425] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n09:168246] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n09:168246] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n09:168246] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n09:168246] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n09:168246] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n09:168246] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n03:46689] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n03:46689] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n03:46689] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n03:46689] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n03:46689] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n03:46689] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n16:166497] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n16:166497] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n16:166497] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n16:166497] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n16:166497] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n16:166497] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n16:166497] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n16:166497] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n08:129890] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n06:154621] *** End of error message *** +[b08n17:171201] *** Process received signal *** +[b08n17:171201] Signal: Segmentation fault (11) +[b08n17:171201] Signal code: (3) +[b08n17:171201] Failing at address: 0xb170000000000a0 +[b08n17:171201] [ 0] [0x1000000504d8] +[b08n17:171201] [ 1] [0x3fffc9db4cc0] +[b08n17:171201] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n17:171201] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n07:117363] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n07:117363] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n17:163820] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n16:165921] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n16:165921] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n16:165921] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n16:165921] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n16:165921] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n16:165921] [ 9] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n04:52703] [ 6] [0x1000000504d8] +[a31n03:158660] [ 1] [0x3fffe5a99770] +[a31n03:158660] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n03:158660] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n03:158660] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n13:122034] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n13:122034] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n13:122034] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n13:122034] [14] [a33n08:147824] *** Process received signal *** +[a33n08:147824] Signal: Segmentation fault (11) +[a33n08:147824] Signal code: (3) +[a33n08:147824] Failing at address: 0x14170000000000a0 +[a33n08:147824] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n11:134750] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n01:153684] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n18:134185] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n14:161621] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n14:161621] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n14:161621] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n17:138143] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n17:138143] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n17:138143] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n17:138143] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n12:12766] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n12:12766] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n12:12766] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n12:12766] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n12:12766] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n17:124165] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n17:124165] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n17:124165] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n17:124165] [14] [b10n16:164551] *** Process received signal *** +[b10n16:164551] Signal: Segmentation fault (11) +[b10n16:164551] Signal code: (3) +[b10n16:164551] Failing at address: 0x14170000000000a0 +[b10n16:164551] [ 0] [0x1000000504d8] +[b18n10:122210] [ 1] [0x3fffcfc60f00] +[b18n10:122210] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n10:122210] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n10:122210] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n10:122210] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n10:122210] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n09:160371] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n09:160371] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n09:160371] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n09:160371] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n09:160371] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n09:160371] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n09:160371] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n09:160371] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n09:160371] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n18:69152] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n11:124433] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n11:124433] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n11:124433] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n11:124433] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n14:177223] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n14:177223] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n14:177223] [ 6] [b16n10:124501] *** Process received signal *** +[b16n10:124501] Signal: Segmentation fault (11) +[b16n10:124501] Signal code: (3) +[b16n10:124501] Failing at address: 0x14170000000000a0 +[b16n10:124501] [ 0] [0x1000000504d8] +[b16n10:124501] [ 1] [0x3ffff0de76d0] +[b16n10:124501] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n10:124501] [ 3] [ 0] [0x1000000504d8] +[a27n01:48804] [ 1] [0x3ffffa1ad900] +[a27n01:48804] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n01:48804] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n01:48804] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n15:123039] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n15:123039] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n12:120406] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n12:120406] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n12:120406] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n12:120406] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n12:120406] *** End of error message *** +[b10n05:173081] *** Process received signal *** +[b10n05:173081] Signal: Segmentation fault (11) +[b10n05:173081] Signal code: (3) +[b10n05:173081] Failing at address: 0x14170000000000a0 +[b10n05:173081] [ 0] [0x1000000504d8] +[b10n05:173081] [ 1] [0x3fffd5450210] +[b10n05:173081] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n05:173081] [ 3] [a34n16:135790] *** Process received signal *** +[a34n16:135790] Signal: Segmentation fault (11) +[a34n16:135790] Signal code: (3) +[a34n16:135790] Failing at address: 0xb170000000000a0 +[a34n16:135790] [ 0] [0x1000000504d8] +[a34n16:135790] [ 1] [0x3fffdc8c5210] +[a34n16:135790] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n16:135790] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n16:135790] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n16:135790] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n16:135790] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n16:135790] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n16:135790] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n16:135790] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n16:135790] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n16:135790] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n16:135790] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n16:135790] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n16:135790] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n16:135790] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n12:164384] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n12:164384] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n12:164384] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n12:164384] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n12:164384] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n13:10197] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n13:171108] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n13:171108] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n13:171108] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n13:171108] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n13:171108] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n13:171108] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n10:155401] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n10:155401] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n10:155401] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n10:155401] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n10:155401] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n10:155401] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n10:155401] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n10:155401] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n16:171169] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n16:171169] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n17:146795] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n17:146795] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n14:08951] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n14:08951] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n14:08951] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n15:177057] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n15:177057] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n15:177057] *** End of error message *** +[b03n12:167678] Signal: Segmentation fault (11) +[b03n12:167678] Signal code: (3) +[b03n12:167678] Failing at address: 0x14170000000000a0 +[b03n12:167678] [ 0] [0x1000000504d8] +[b03n12:167678] [ 1] [0x3fffdf8dad30] +[b03n12:167678] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n12:167678] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n11:117474] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n11:117474] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n11:117474] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n11:117474] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n11:117474] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n09:123815] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n09:123815] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n09:123815] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n09:123815] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n03:121811] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n03:121811] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n03:121811] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n03:121811] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n16:116524] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n16:116524] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n16:116524] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n16:116524] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n16:116524] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n05:135167] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n05:135167] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n03:147538] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n03:147538] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n03:147538] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n03:147538] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n03:147538] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n03:147538] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n03:147538] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n03:147538] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n03:147538] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n03:147538] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n03:147538] [14] [0x3fffec1ac640] +[b07n17:162360] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n17:162360] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n17:162360] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n17:162360] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n17:162360] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n17:162360] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n17:162360] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n17:162360] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n17:162360] [10] [0x3ffff3214d20] +[a33n10:140412] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n10:140412] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n10:140412] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n10:140412] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n10:162609] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n10:162609] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n10:162609] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n10:162609] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n10:162609] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n01:145732] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n01:145732] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n01:145732] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n01:145732] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n01:138008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n01:138008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n01:138008] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n01:141342] *** End of error message *** +[a28n09:03928] *** Process received signal *** +[a28n09:03928] Signal: Segmentation fault (11) +[a28n09:03928] Signal code: (3) +[a28n09:03928] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n13:165389] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n13:165389] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n13:165389] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n13:165389] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n13:165389] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n16:127469] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n16:127469] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n16:127469] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n16:127469] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n16:127469] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n16:127469] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n16:127469] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n16:127469] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n16:127469] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n16:127469] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n16:127469] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n16:127469] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n07:118165] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n10:176308] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n10:176308] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n10:176308] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n14:128869] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n14:128869] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n14:128869] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n14:128869] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n14:128869] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n17:119466] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n17:119466] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n17:119466] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n09:143901] [ 5] [0x3fffd8e78eb0] +[b05n09:106791] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n09:106791] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n09:106791] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n09:106791] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n09:106791] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n09:106791] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n09:106791] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n09:106791] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n09:106791] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n09:106791] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n09:106791] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n09:106791] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n09:106791] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n09:106791] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n14:133633] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n10:10927] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n10:10927] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n10:10927] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n10:10927] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n10:10927] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n14:169523] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n14:169523] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n14:169523] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n04:44760] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n04:44760] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n13:123707] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n13:123707] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n12:119807] *** End of error message *** +[11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n01:167302] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n01:167302] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n01:167302] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n01:167302] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n01:179261] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n01:179261] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n02:119535] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n02:119535] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n18:179701] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n18:179701] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n18:179701] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n18:179701] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n04:117356] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n04:117356] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n04:117356] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n04:117356] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n16:53795] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n16:53795] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n16:53795] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n01:141817] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n01:141817] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n01:141817] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n12:123563] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n12:123563] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n12:123563] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n12:123563] [13] [0x1000000504d8] +[a36n07:153176] [ 1] [0x3fffd1787e00] +[a36n07:153176] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n07:153176] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n07:153176] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n04:176001] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n04:176001] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n13:163196] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n18:131583] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n18:131583] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n18:131583] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n18:131583] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n18:131583] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n18:131583] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n18:131583] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n18:131583] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n18:131583] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n18:131583] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n13:156922] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n13:156922] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n13:156922] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n13:156922] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n11:130805] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n11:130805] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n11:148676] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n11:148676] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n18:28218] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n18:28218] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n18:28218] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n04:169914] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n04:169914] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n04:169914] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n04:169914] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n08:144856] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n08:144856] *** End of error message *** +[a28n13:176368] *** Process received signal *** +[a28n13:176368] Signal: Segmentation fault (11) +[a28n13:176368] Signal code: (3) +[a28n13:176368] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n11:120585] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n11:120585] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n11:120585] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n17:126525] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n17:126525] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n17:126525] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n17:126525] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n12:177121] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n13:166927] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n13:166927] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n13:166927] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n13:166927] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n13:166927] [13] [a31n16:162630] *** Process received signal *** +[a31n16:162630] Signal: Segmentation fault (11) +[a31n16:162630] Signal code: (3) +[a31n16:162630] Failing at address: 0x14170000000000a0 +[a31n16:162630] [ 0] [0x1000000504d8] +[a31n16:162630] [ 1] [0x3fffd9687880] +[a31n16:162630] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n08:08954] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n08:08954] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n08:08954] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n08:08954] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n10:01256] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n10:01256] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n10:01256] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n10:01256] [13] [b03n15:85940] *** Process received signal *** +[b03n15:85940] Signal: Segmentation fault (11) +[b03n15:85940] Signal code: (3) +[b03n15:85940] Failing at address: 0x14170000000000a0 +[b03n15:85940] [ 0] [0x1000000504d8] +[b03n15:85940] [ 1] [0x3fffd7f7a810] +[b03n15:85940] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n15:85940] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n07:132091] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n07:132091] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n07:132091] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n07:132091] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n12:115641] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n12:115641] [14] [b21n09:120695] *** Process received signal *** +[b21n09:120695] Signal: Segmentation fault (11) +[b21n09:120695] Signal code: (3) +[b21n09:120695] Failing at address: 0x14170000000000a0 +[b21n09:120695] [ 0] [0x1000000504d8] +[b21n09:120695] [ 1] [0x3ffff924da90] +[b21n09:120695] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n09:120695] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n09:120695] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n07:125449] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n07:125449] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n10:126192] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n10:126192] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n10:126192] [ 5] [b01n09:173750] *** Process received signal *** +[b01n09:173750] Signal: Segmentation fault (11) +[b01n09:173750] Signal code: (3) +[b01n09:173750] Failing at address: 0xb170000000000a0 +[b01n09:173750] [ 0] [b21n10:130583] *** Process received signal *** +[0x1000000504d8] +[b05n02:176049] [ 1] [0x3fffc25fc180] +[b05n02:176049] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n02:176049] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n02:176049] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n02:176049] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n03:10290] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n03:10290] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n03:10290] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n09:119550] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n09:119550] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n11:01175] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n11:01175] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n11:01175] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n11:01175] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n11:01175] [13] [b05n15:176425] *** Process received signal *** +[b05n15:176425] Signal: Segmentation fault (11) +[b05n15:176425] Signal code: (3) +[b05n15:176425] Failing at address: 0x5170000000000a0 +[b05n15:176425] [ 0] [0x1000000504d8] +[b05n15:176425] [ 1] [0x3ffff3bd6ae0] +[b05n15:176425] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n15:176425] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n15:176425] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n16:173909] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n16:173909] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n16:173909] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n16:173909] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n16:173909] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n16:173909] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n16:173909] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n16:173909] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n16:173909] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n16:173909] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n16:173909] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n13:01301] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n13:01301] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n13:01301] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n13:01301] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n13:01301] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n16:171921] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n16:171921] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n08:124724] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n08:124724] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n08:124724] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n08:124724] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n08:124724] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n08:124724] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n08:124724] *** End of error message *** +[b15n10:122761] *** Process received signal *** +[b15n10:122761] Signal: Segmentation fault (11) +[b15n10:122761] Signal code: (3) +[b15n10:122761] Failing at address: 0xb170000000000a0 +[b15n10:122761] [ 0] [b04n15:177862] [ 0] [0x1000000504d8] +[b04n15:177862] [ 1] [0x3fffd15f34d0] +[b04n15:177862] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n15:177862] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n15:177862] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n18:164009] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n18:164009] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n18:164009] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n18:164009] [10] [a30n16:153290] *** Process received signal *** +[a30n16:153290] Signal: Segmentation fault (11) +[a30n16:153290] Signal code: (3) +[a30n16:153290] Failing at address: 0xb170000000000a0 +[a30n16:153290] [ 0] [0x1000000504d8] +[a30n16:153290] [ 1] [0x3ffff73c6aa0] +[a30n16:153290] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n16:153290] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n02:166748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n02:166748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n02:166748] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n02:166748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n11:137871] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n11:137871] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n11:137871] [11] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n16:127354] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n16:127354] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n15:161755] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n15:161755] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n15:161755] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n17:150932] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n17:150932] [ 8] [b14n08:127444] [ 0] [0x1000000504d8] +[b14n08:127444] [ 1] [0x3fffc4fd1130] +[b14n08:127444] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n08:127444] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n08:127444] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n08:127444] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n08:127444] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n02:49109] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n02:49109] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n02:49109] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n02:49109] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n11:02685] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n11:02685] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n11:02685] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n11:02685] [ 6] [a26n02:74429] *** Process received signal *** +[a26n02:74429] Signal: Segmentation fault (11) +[a26n02:74429] Signal code: (3) +[a26n02:74429] Failing at address: 0xb170000000000a0 +[a26n02:74429] [ 0] [0x1000000504d8] +[a26n02:74429] [ 1] [0x3fffd96b70e0] +[a26n02:74429] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n02:74429] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n02:74429] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n02:74429] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n02:74429] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n11:156153] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n11:156153] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n11:156153] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n11:156153] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n14:122442] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n14:122442] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n14:122442] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n15:151253] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n15:151253] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n15:151253] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n15:151253] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n10:161721] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n10:161721] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n10:161721] *** End of error message *** +[a36n09:140255] *** Process received signal *** +[a36n09:140255] Signal: Segmentation fault (11) +[a36n09:140255] Signal code: (3) +[a36n09:140255] Failing at address: 0x5170000000000a0 +[a36n09:140255] [ 0] [0x1000000504d8] +[a36n09:140255] [ 1] [0x3fffc8cf3ce0] +[a36n09:140255] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n09:140255] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n13:06927] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n13:06927] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n13:06927] [ 6] [a33n09:147795] *** Process received signal *** +[a33n09:147795] Signal: Segmentation fault (11) +[a33n09:147795] Signal code: (3) +[a33n09:147795] Failing at address: 0x14170000000000a0 +[a33n09:147795] [ 0] [0x1000000504d8] +[a33n09:147795] [ 1] [0x3ffff5765ab0] +[a33n09:147795] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n09:147795] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n04:13083] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n04:13083] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n04:13083] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n04:13083] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n04:13083] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n04:13083] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n04:13083] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n04:13083] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n04:13083] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n04:13083] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n04:13083] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n18:164801] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n18:164801] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n18:164801] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n18:164801] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n16:124795] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n16:124795] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n16:124795] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n16:124795] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n16:124795] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n08:172428] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n18:10725] *** Process received signal *** +[a30n18:10725] Signal: Segmentation fault (11) +[a30n18:10725] Signal code: (3) +[a30n18:10725] Failing at address: 0x14170000000000a0 +[a30n18:10725] [ 0] [0x1000000504d8] +[a30n18:10725] [ 1] [0x3fffe4cbfb20] +[a30n18:10725] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n18:10725] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n18:10725] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n18:10725] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n18:10725] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n18:10725] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n13:133628] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n13:133628] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n13:133628] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n03:125239] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n03:125239] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n05:162570] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n05:162570] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n05:162570] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n05:162570] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n06:130542] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n06:130542] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n08:118552] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n08:118552] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n08:118552] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n11:164998] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n11:164998] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n11:164998] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n05:120229] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n05:120229] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n05:120229] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n12:160187] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n12:160187] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n12:160187] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n17:119581] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n17:119581] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n17:119581] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n15:166121] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n15:166121] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n04:06832] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n04:06832] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n03:126188] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n03:126188] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n03:126188] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n03:126188] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n03:126188] *** End of error message *** +[a32n05:154190] *** Process received signal *** +[a32n05:154190] Signal: Segmentation fault (11) +[a32n05:154190] Signal code: (3) +[a32n05:154190] Failing at address: 0x5170000000000a0 +[a32n05:154190] [ 0] [0x1000000504d8] +[a32n05:154190] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n15:154384] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n15:154384] [ 6] [a27n05:53375] *** Process received signal *** +[a27n05:53375] Signal: Segmentation fault (11) +[a27n05:53375] Signal code: (3) +[a27n05:53375] Failing at address: 0x14170000000000a0 +[a27n05:53375] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n01:128003] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n01:128003] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n01:128003] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n01:128003] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n01:128003] [14] [b18n16:124959] *** Process received signal *** +[b18n16:124959] Signal: Segmentation fault (11) +[b18n16:124959] Signal code: (3) +[b18n16:124959] Failing at address: 0xb170000000000a0 +[b18n16:124959] [ 0] [0x1000000504d8] +[b18n16:124959] [ 1] [0x3fffcaf7f470] +[b18n16:124959] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n16:124959] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n06:38120] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n06:38120] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n06:38120] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n06:38120] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n06:38120] [ 9] [0x1000000504d8] +[a31n15:155431] [ 1] [0x3fffda066750] +[a31n15:155431] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n15:155431] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n15:155431] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n15:155431] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n15:155431] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n09:161233] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n09:161233] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n09:161233] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n09:161233] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n09:161233] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n09:161233] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n03:164496] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n03:164496] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n03:164496] [ 7] [0x1000000504d8] +[b22n14:116415] [ 1] [0x3fffd4ce6010] +[b22n14:116415] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n14:116415] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n14:116415] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n08:76225] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n08:76225] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n08:76225] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n08:76225] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n08:76225] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n08:76225] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n08:76225] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n08:76225] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n08:76225] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n08:76225] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n08:76225] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n08:76225] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n14:150132] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n14:150132] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n14:150132] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n14:150132] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n15:148165] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n15:148165] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n11:23063] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n11:23063] *** End of error message *** +[b13n10:134735] *** Process received signal *** +[b13n10:134735] Signal: Segmentation fault (11) +[b13n10:134735] Signal code: (3) +[b13n10:134735] Failing at address: 0xb170000000000a0 +[b13n10:134735] [ 0] [0x1000000504d8] +[b13n10:134735] [ 1] [0x3fffd1a3e7c0] +[b13n10:134735] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n10:134735] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n05:122273] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n05:122273] *** End of error message *** +[0x3ffffe862210] +[b08n02:162034] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n02:162034] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n02:162034] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n02:162034] [ 5] [b02n09:166572] *** Process received signal *** +[b02n09:166572] Signal: Segmentation fault (11) +[b02n09:166572] Signal code: (3) +[b02n09:166572] Failing at address: 0x14170000000000a0 +[b02n09:166572] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n11:128958] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n11:128958] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n11:128958] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n11:128958] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n11:128958] [10] [b05n16:168049] *** Process received signal *** +[b05n16:168049] Signal: Segmentation fault (11) +[b05n16:168049] Signal code: (3) +[b05n16:168049] Failing at address: 0x5170000000000a0 +[b05n16:168049] [ 0] [0x1000000504d8] +[b05n16:168049] [ 1] [0x3ffffef4b900] +[b05n16:168049] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n16:168049] [ 3] [b18n18:121304] *** Process received signal *** +[b18n18:121304] Signal: Segmentation fault (11) +[b18n18:121304] Signal code: (3) +[b18n18:121304] Failing at address: 0xb170000000000a0 +[b16n12:134493] *** Process received signal *** +[b16n12:134493] Signal: Segmentation fault (11) +[b16n12:134493] Signal code: (3) +[b16n12:134493] Failing at address: 0x14170000000000a0 +[b16n12:134493] [ 0] [0x1000000504d8] +[b16n12:134493] [ 1] [0x3ffff373c270] +[b16n12:134493] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n12:134493] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n12:134493] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n09:152063] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n09:152063] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n04:153964] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n04:153964] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n04:153964] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n04:153964] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n04:153964] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n04:153964] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n04:153964] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n04:153964] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n04:153964] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n12:145267] *** End of error message *** +[b13n07:133377] *** Process received signal *** +[b13n07:133377] Signal: Segmentation fault (11) +[b13n07:133377] Signal code: (3) +[b13n07:133377] Failing at address: 0xb170000000000a0 +[b13n07:133377] [ 0] [0x1000000504d8] +[b13n07:133377] [ 1] [0x3fffe6b393a0] +[b13n07:133377] [ 2] [a30n11:163262] *** Process received signal *** +[a30n11:163262] Signal: Segmentation fault (11) +[a30n11:163262] Signal code: (3) +[a30n11:163262] Failing at address: 0x14170000000000a0 +[a30n11:163262] [ 0] [0x1000000504d8] +[a30n11:163262] [ 1] [0x3fffed49ce30] +[a30n11:163262] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n11:163262] [ 3] [0x1000000504d8] +[b17n18:126908] [ 1] [0x3fffc729cf70] +[b17n18:126908] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n18:126908] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n18:126908] [ 4] [b02n18:03315] *** Process received signal *** +[b02n18:03315] Signal: Segmentation fault (11) +[b02n18:03315] Signal code: (3) +[b02n18:03315] Failing at address: 0xb170000000000a0 +[b02n18:03315] [ 0] [0x1000000504d8] +[b02n18:03315] [ 1] [0x3fffcf354300] +[b02n18:03315] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n18:03315] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n06:155089] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n06:155089] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n06:155089] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n06:155089] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n05:148379] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n05:148379] *** End of error message *** +[b09n04:177596] *** Process received signal *** +[b09n04:177596] Signal: Segmentation fault (11) +[b09n04:177596] Signal code: (3) +[b09n04:177596] Failing at address: 0xb170000000000a0 +[b09n04:177596] [ 0] [0x1000000504d8] +[b09n04:177596] [ 1] [0x3fffe3952390] +[b09n04:177596] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n04:177596] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n04:177596] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n05:173848] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n05:173848] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n05:173848] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n17:164298] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n17:164298] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n17:164298] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n17:164298] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n17:164298] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n17:164298] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n17:164298] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n17:164298] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n17:164298] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n17:164298] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n17:164298] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n13:131758] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n13:131758] [ 5] [b08n01:171784] *** Process received signal *** +[b08n01:171784] Signal: Segmentation fault (11) +[b08n01:171784] Signal code: (3) +[b08n01:171784] Failing at address: 0x14170000000000a0 +[b08n01:171784] [ 0] [0x1000000504d8] +[b08n01:171784] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n08:138179] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n08:138179] *** End of error message *** +[0x3fffd12eb140] +[a28n17:168164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n17:168164] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n17:168164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n17:168164] [ 5] [a26n09:62481] *** Process received signal *** +[a26n09:62481] Signal: Segmentation fault (11) +[a26n09:62481] Signal code: (3) +[a26n09:62481] Failing at address: 0x14170000000000a0 +[a26n09:62481] [ 0] [0x1000000504d8] +[a26n09:62481] [ 1] [0x3fffce9b7950] +[a26n09:62481] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n09:62481] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n16:128440] *** End of error message *** +[b12n14:139846] *** Process received signal *** +[b12n14:139846] Signal: Segmentation fault (11) +[b12n14:139846] Signal code: (3) +[b12n14:139846] Failing at address: 0x5170000000000a0 +[b12n14:139846] [ 0] [0x1000000504d8] +[b12n14:139846] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n10:169682] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n10:169682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n10:169682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n10:169682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n10:169682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n10:169682] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n10:169682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n10:169682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n10:169682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n10:169682] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n12:156916] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n12:156916] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n12:156916] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n12:156916] [10] [0x3fffc0c01b30] +[b16n13:122187] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n13:122187] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n13:122187] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n08:164750] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n08:164750] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n08:164750] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n10:135855] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n10:135855] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n10:135855] [ 7] [b01n13:177641] *** Process received signal *** +[b01n13:177641] Signal: Segmentation fault (11) +[b01n13:177641] Signal code: (3) +[b01n13:177641] Failing at address: 0x14170000000000a0 +[b01n13:177641] [ 0] [0x1000000504d8] +[b01n13:177641] [ 1] [0x3fffe0b7f8e0] +[b01n13:177641] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n13:177641] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n13:177641] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n14:152749] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n14:152749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n14:152749] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n11:160346] [13] [a31n02:158418] *** Process received signal *** +[a31n02:158418] Signal: Segmentation fault (11) +[a31n02:158418] Signal code: (3) +[a31n02:158418] Failing at address: 0xb170000000000a0 +[a31n02:158418] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n13:138089] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n15:177821] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n15:177821] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n15:177821] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n15:177821] [ 8] [b14n06:127277] *** Process received signal *** +[b14n06:127277] Signal: Segmentation fault (11) +[b14n06:127277] Signal code: (3) +[b14n06:127277] Failing at address: 0xb170000000000a0 +[b14n06:127277] [ 0] [b08n03:175066] *** Process received signal *** +[b08n03:175066] Signal: Segmentation fault (11) +[b08n03:175066] Signal code: (3) +[b08n03:175066] Failing at address: 0x14170000000000a0 +[b08n03:175066] [ 0] [0x1000000504d8] +[b08n03:175066] [ 1] [0x3fffe0418670] +[b08n03:175066] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n03:175066] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n08:141875] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n08:141875] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n08:141875] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n15:32401] [ 4] [b03n09:173062] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n09:173062] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n06:164017] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n06:164017] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n06:164017] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n06:164017] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n14:08400] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n14:08400] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n14:08400] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n18:176811] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n18:176811] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n18:176811] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n18:176811] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n18:176811] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n18:176811] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n18:176811] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n02:163320] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n02:163320] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n02:163320] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n02:163320] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n02:163320] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n02:163320] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n17:118573] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n17:118573] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n17:118573] [ 9] [a33n02:167597] [ 0] [0x1000000504d8] +[a33n02:167597] [ 1] [0x3fffc4d492d0] +[a33n02:167597] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n02:167597] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n11:162471] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n11:162471] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n08:118580] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n08:118580] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n08:118580] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n18:173977] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n18:173977] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n14:119483] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n14:119483] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n14:119483] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n14:119483] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n14:119483] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n14:119483] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n14:119483] [10] [0x3fffe20f4bd0] +[b21n15:119252] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n15:119252] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n15:119252] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n15:119252] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n02:164493] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n02:164493] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n04:126859] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n04:126859] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n13:158867] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n13:158867] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n13:158867] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n06:133957] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n06:133957] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n06:133957] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n06:133957] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n10:118388] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n10:118388] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n10:118388] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n18:140388] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n18:140388] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n18:140388] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n18:140388] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n18:140388] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n18:140388] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n18:140388] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n18:140388] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n18:140388] [13] [b10n14:168376] *** Process received signal *** +[b10n14:168376] Signal: Segmentation fault (11) +[b10n14:168376] Signal code: (3) +[b10n14:168376] Failing at address: 0xb170000000000a0 +[b10n14:168376] [ 0] [0x1000000504d8] +[b10n14:168376] [ 1] [0x3fffd7cd4030] +[b10n14:168376] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n14:168376] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n01:173783] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n01:173783] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n01:173783] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n01:173783] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n01:173783] *** End of error message *** +[a33n16:149064] *** Process received signal *** +[a33n16:149064] Signal: Segmentation fault (11) +[a33n16:149064] Signal code: (3) +[a33n16:149064] Failing at address: 0x14170000000000a0 +[a33n16:149064] [ 0] [0x1000000504d8] +[a33n16:149064] [ 1] [0x3ffff3099b50] +[a33n16:149064] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n16:149064] [ 3] [0x1000000504d8] +[b06n14:173894] [ 1] [0x3fffd590f720] +[b06n14:173894] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n14:173894] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n14:173894] [ 4] [b21n13:117510] *** Process received signal *** +[b21n13:117510] Signal: Segmentation fault (11) +[b21n13:117510] Signal code: (3) +[b21n13:117510] Failing at address: 0xb170000000000a0 +[b21n13:117510] [ 0] [a30n02:155384] *** Process received signal *** +[a30n02:155384] Signal: Segmentation fault (11) +[a30n02:155384] Signal code: (3) +[a30n02:155384] Failing at address: 0x14170000000000a0 +[a30n02:155384] [ 0] [0x1000000504d8] +[a30n02:155384] [ 1] [0x3fffd74501a0] +[a30n02:155384] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n06:167372] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n06:167372] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n06:167372] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n06:167372] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n06:167372] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n09:137776] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n09:174689] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n09:174689] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n09:174689] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n09:174689] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n09:174689] [ 8] [b06n11:160797] *** Process received signal *** +[b06n11:160797] Signal: Segmentation fault (11) +[b06n11:160797] Signal code: (3) +[b06n11:160797] Failing at address: 0x5170000000000a0 +[b06n11:160797] [ 0] [0x1000000504d8] +[b06n11:160797] [ 1] [0x3fffca1ab9f0] +[b06n11:160797] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n14:136660] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n14:136660] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n14:136660] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n14:136660] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n05:126783] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n05:126783] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n05:126783] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n05:126783] *** End of error message *** +[b09n16:173548] *** Process received signal *** +[b09n16:173548] Signal: Segmentation fault (11) +[b09n16:173548] Signal code: (3) +[b09n16:173548] Failing at address: 0x14170000000000a0 +[b09n16:173548] [ 0] [0x1000000504d8] +[b09n16:173548] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n18:08429] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n18:08429] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n18:08429] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n18:08429] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n18:08429] [ 9] [b08n06:161721] *** Process received signal *** +[b08n06:161721] Signal: Segmentation fault (11) +[b08n06:161721] Signal code: (3) +[b08n06:161721] Failing at address: 0xb170000000000a0 +[b08n06:161721] [ 0] [0x1000000504d8] +[b08n06:161721] [ 1] [0x3fffe9ebc460] +[b08n06:161721] [ 2] [b12n06:135851] *** Process received signal *** +[b12n06:135851] Signal: Segmentation fault (11) +[b12n06:135851] Signal code: (3) +[b12n06:135851] Failing at address: 0x14170000000000a0 +[b12n06:135851] [ 0] [0x1000000504d8] +[b12n06:135851] [ 1] [0x3fffef3cf040] +[b12n06:135851] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n06:135851] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n14:21318] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n14:21318] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n14:21318] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n14:21318] [ 7] [a30n12:148383] *** Process received signal *** +[a30n12:148383] Signal: Segmentation fault (11) +[a30n12:148383] Signal code: (3) +[a30n12:148383] Failing at address: 0x14170000000000a0 +[a30n12:148383] [ 0] [0x1000000504d8] +[a30n12:148383] [ 1] [0x3fffe2dd1480] +[a30n12:148383] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n12:148383] [ 3] [b03n18:177223] *** Process received signal *** +[b03n18:177223] Signal: Segmentation fault (11) +[b03n18:177223] Signal code: (3) +[b03n18:177223] Failing at address: 0xb170000000000a0 +[b03n18:177223] [ 0] [0x1000000504d8] +[b03n18:177223] [ 1] [0x3fffcf337cb0] +[b03n18:177223] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n18:177223] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n14:178324] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n14:178324] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n14:178324] [ 9] [b12n05:138197] *** Process received signal *** +[b12n05:138197] Signal: Segmentation fault (11) +[b12n05:138197] Signal code: (3) +[b12n05:138197] Failing at address: 0xb170000000000a0 +[b12n05:138197] [ 0] [0x1000000504d8] +[b12n05:138197] [ 1] [0x3fffc429cc70] +[b12n05:138197] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n05:138197] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n06:162918] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n06:162918] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n09:149101] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n09:149101] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n09:149101] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n09:149101] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n09:149101] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n07:03373] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n07:03373] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n07:03373] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n03:46689] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n03:46689] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n03:46689] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n03:46689] *** End of error message *** +[a32n18:138475] *** Process received signal *** +[a32n18:138475] Signal: Segmentation fault (11) +[a32n18:138475] Signal code: (3) +[a32n18:138475] Failing at address: 0xb170000000000a0 +[a32n18:138475] [ 0] [0x1000000504d8] +[a32n18:138475] [ 1] [0x3ffff3c77980] +[a32n18:138475] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n16:166497] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n18:07424] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n18:07424] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n18:07424] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n18:07424] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n18:07424] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n18:07424] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n18:07424] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n18:07424] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n18:07424] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n09:168246] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n09:168246] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n09:168246] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n17:171200] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n17:171200] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n17:171200] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n17:171200] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n17:171200] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n16:165922] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n16:165922] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n04:52703] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n04:52703] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n04:52703] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n04:52703] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n04:52703] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n04:52703] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n04:52703] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n04:52703] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n04:52703] *** End of error message *** +[b21n07:117359] *** Process received signal *** +[b21n18:118682] *** Process received signal *** +[b21n18:118682] Signal: Segmentation fault (11) +[b21n18:118682] Signal code: (3) +[b21n18:118682] Failing at address: 0xb170000000000a0 +[b21n18:118682] [ 0] [0x1000000504d8] +[b21n18:118682] [ 1] [0x3fffe15eda30] +[b21n18:118682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n18:118682] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n18:118682] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n17:163820] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n13:122031] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n13:122031] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n13:122031] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n13:122031] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n17:149905] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n17:149905] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n17:149905] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n17:149905] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n17:149905] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n17:149905] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n17:164561] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n17:164561] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n08:156370] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n08:156370] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n08:156370] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n08:156370] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n11:134747] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n11:134747] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n11:134747] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n17:124168] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n17:124168] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n09:160371] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n09:160371] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n18:69152] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n18:69152] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n18:69152] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n18:69152] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n18:69152] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n18:69152] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n11:124433] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n14:177223] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n14:177223] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n14:177223] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n14:177223] [10] [b16n10:124502] *** Process received signal *** +[b16n10:124502] Signal: Segmentation fault (11) +[b16n10:124502] Signal code: (3) +[b16n10:124502] Failing at address: 0x14170000000000a0 +[b16n10:124502] [ 0] [0x1000000504d8] +[b16n10:124502] [ 1] [0x3fffe5fb2030] +[b16n10:124502] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n10:124502] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n01:48804] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n01:48804] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n01:48804] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n01:48804] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n01:48804] [ 9] [a02n03:38501] *** Process received signal *** +[a02n03:38501] Signal: Segmentation fault (11) +[a02n03:38501] Signal code: (3) +[a02n03:38501] Failing at address: 0x14170000000000a0 +[a02n03:38501] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n15:123042] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n15:123042] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n15:123042] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n15:123042] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n15:123042] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n03:151884] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n03:151884] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n03:151884] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n03:151884] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n14:08952] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n14:08952] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n14:08952] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n06:148312] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n06:148312] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n06:148312] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n06:148312] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n06:148312] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n13:171108] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n08:129890] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n08:129890] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n15:177056] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n15:177056] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n12:167677] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n12:167677] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n02:122557] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n02:122557] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n02:122557] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n17:146795] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n17:146795] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n17:146795] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n17:146795] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n17:146795] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n03:147535] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n03:147535] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n03:147535] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n03:147535] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n03:147535] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n03:147535] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n03:147535] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n03:147535] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n03:147535] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n03:147535] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n03:147535] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n03:147535] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n16:116524] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n16:116524] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n16:116524] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n16:116524] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n16:116524] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n16:116524] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n05:135164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n05:135164] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n05:135164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n01:145732] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n01:145732] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n01:145732] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n01:145732] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n13:165390] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n13:165390] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n13:165390] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n13:165390] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n13:165390] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n07:118168] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n07:118168] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n07:118168] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n07:118168] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n07:118168] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n03:121811] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n16:127468] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n16:127468] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n16:127468] *** End of error message *** +[a28n09:03928] [ 0] [0x1000000504d8] +[a28n09:03928] [ 1] [0x3fffcdc210b0] +[a28n09:03928] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n09:03928] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n09:03928] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n14:128869] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n14:128869] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n14:128869] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n14:128869] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n10:176308] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n10:176308] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n10:176308] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n17:119466] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n17:119466] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n17:119466] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n17:119466] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n07:156491] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n07:156491] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n09:106788] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n09:106788] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n09:106788] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n09:106788] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n09:106788] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n09:106788] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n09:106788] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n09:106788] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n09:106788] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n09:106788] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n09:106788] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n14:133633] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n14:133633] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n14:133633] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n14:133633] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n14:169524] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n14:169524] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n14:169524] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n14:169524] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n14:169524] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n10:10924] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n10:10924] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n10:10924] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n10:10924] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n10:10924] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n05:135124] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n05:135124] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n05:135124] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n05:135124] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n05:135124] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n01:179261] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n01:179261] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n01:179261] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n01:179261] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n01:179261] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n01:179261] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n01:179261] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n18:179701] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n18:179701] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n18:179701] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n01:141817] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n01:141817] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n02:119535] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n02:119535] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n02:119535] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n02:119535] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n16:53795] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n16:53795] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n16:53795] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n16:53795] [13] [b22n01:122015] *** Process received signal *** +[b22n01:122015] Signal: Segmentation fault (11) +[b22n01:122015] Signal code: (3) +[b22n01:122015] Failing at address: 0x14170000000000a0 +[b22n01:122015] [ 0] [0x1000000504d8] +[b22n01:122015] [ 1] [0x3ffffe2f97a0] +[b22n01:122015] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n01:122015] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n01:122015] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n12:123563] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n12:123563] *** End of error message *** +[b20n16:124665] *** Process received signal *** +[b20n16:124665] Signal: Segmentation fault (11) +[b20n16:124665] Signal code: (3) +[b20n16:124665] Failing at address: 0xb170000000000a0 +[b20n16:124665] [ 0] [0x1000000504d8] +[b20n16:124665] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n09:155761] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n09:155761] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n09:155761] [ 6] [b10n12:165845] *** Process received signal *** +[b10n12:165845] Signal: Segmentation fault (11) +[b10n12:165845] Signal code: (3) +[b10n12:165845] Failing at address: 0xb170000000000a0 +[b10n12:165845] [ 0] [0x1000000504d8] +[b10n12:165845] [ 1] [0x3fffd0230350] +[b10n12:165845] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n12:165845] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n12:165845] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n12:165845] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n12:165845] [ 6] [a36n06:143011] *** Process received signal *** +[a36n06:143011] Signal: Segmentation fault (11) +[a36n06:143011] Signal code: (3) +[a36n06:143011] Failing at address: 0x5170000000000a0 +[a36n06:143011] [ 0] [0x1000000504d8] +[a36n06:143011] [ 1] [0x3fffff014770] +[a36n06:143011] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n06:143011] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n07:153176] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n07:153176] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n07:153176] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n07:153176] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n07:153176] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n07:153176] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n04:176001] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n04:176001] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n04:176001] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n04:176001] [10] [a36n13:156920] *** Process received signal *** +[a36n13:156920] Signal: Segmentation fault (11) +[a36n13:156920] Signal code: (3) +[a36n13:156920] Failing at address: 0xb170000000000a0 +[a36n13:156920] [ 0] [0x1000000504d8] +[a36n13:156920] [ 1] [0x3ffffae9cf20] +[a36n13:156920] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n13:156920] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n18:28218] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n18:28218] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n18:28218] [14] [a33n04:169911] *** Process received signal *** +[a33n04:169911] Signal: Segmentation fault (11) +[a33n04:169911] Signal code: (3) +[a33n04:169911] Failing at address: 0xb170000000000a0 +[a33n04:169911] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n08:147821] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n08:147821] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n08:147821] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n08:147821] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n08:147821] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n08:147821] [ 9] [a28n13:176368] [ 0] [0x1000000504d8] +[a28n13:176368] [ 1] [0x3fffc14a5a70] +[a28n13:176368] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n13:176368] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n13:176368] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n13:176368] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n08:08954] *** End of error message *** +[b21n11:120585] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n11:120585] *** End of error message *** +[b13n14:126466] *** Process received signal *** +[b13n14:126466] Signal: Segmentation fault (11) +[b13n14:126466] Signal code: (3) +[b13n14:126466] Failing at address: 0xb170000000000a0 +[b13n14:126466] [ 0] [0x1000000504d8] +[b13n14:126466] [ 1] [0x3fffe19a6da0] +[b13n14:126466] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n14:126466] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n14:126466] [ 4] [a31n04:158633] *** Process received signal *** +[a31n04:158633] Signal: Segmentation fault (11) +[a31n04:158633] Signal code: (3) +[a31n04:158633] Failing at address: 0x14170000000000a0 +[a31n04:158633] [ 0] [0x1000000504d8] +[a31n04:158633] [ 1] [0x3fffe1fd32a0] +[a31n04:158633] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n04:158633] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n13:166927] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n13:166927] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n16:162627] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n16:162627] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n10:01256] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n10:01256] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n13:123707] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n13:123707] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n13:123707] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n07:168965] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n07:168965] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n15:85940] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n15:85940] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n15:85940] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n07:132091] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n07:132091] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n12:115641] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n09:120696] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n09:120696] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n09:120696] [10] [0x1000000504d8] +[b01n09:173750] [ 1] [0x3fffd45562a0] +[b01n09:173750] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n09:173750] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n09:173750] [ 4] [b21n10:130582] *** Process received signal *** +[b21n10:130582] Signal: Segmentation fault (11) +[b21n10:130582] Signal code: (3) +[b21n10:130582] Failing at address: 0x14170000000000a0 +[b21n10:130582] [ 0] [0x1000000504d8] +[b21n10:130582] [ 1] [0x3ffff851a790] +[b21n10:130582] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n10:130582] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n02:176049] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n02:176049] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n02:176049] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n02:176049] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n02:176049] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n02:176049] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n02:176049] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n02:176049] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n02:176049] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n03:10290] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n03:10290] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n03:10290] [11] [b15n05:120331] *** Process received signal *** +[b15n05:120331] Signal: Segmentation fault (11) +[b15n05:120331] Signal code: (3) +[b15n05:120331] Failing at address: 0x14170000000000a0 +[b15n05:120331] [ 0] [0x1000000504d8] +[b15n05:120331] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n11:01175] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n11:01175] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n09:119547] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n09:119547] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n09:119547] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n09:119547] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n09:119547] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n16:173910] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n16:173910] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n16:173910] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n16:173910] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n16:173910] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n16:173910] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n16:173910] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n16:173910] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n16:173910] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n16:173910] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n16:173910] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n16:173910] *** End of error message *** +[b20n08:124725] *** Process received signal *** +[b20n08:124725] Signal: Segmentation fault (11) +[b20n08:124725] Signal code: (3) +[b20n08:124725] Failing at address: 0x14170000000000a0 +[b20n08:124725] [ 0] [0x1000000504d8] +[b20n08:124725] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n13:01302] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n13:01302] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n13:01302] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n13:01302] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n13:01302] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n13:01302] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n13:01302] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n18:164009] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n18:164009] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n18:164009] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n18:164009] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n15:177862] [ 5] [b08n16:171919] *** Process received signal *** +[b08n16:171919] Signal: Segmentation fault (11) +[b08n16:171919] Signal code: (3) +[b08n16:171919] Failing at address: 0x14170000000000a0 +[b08n16:171919] [ 0] [0x1000000504d8] +[b08n16:171919] [ 1] [0x3fffd54c8b10] +[b08n16:171919] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n16:171919] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n01:164965] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n01:164965] *** End of error message *** +[0x1000000504d8] +[b10n16:164551] [ 1] [0x3fffcb737f90] +[b10n16:164551] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n16:164551] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n11:137871] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n11:137871] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n11:137871] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[0x1000000504d8] +[b02n02:166745] [ 1] [0x3fffc907cd30] +[b02n02:166745] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n02:166745] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n02:166745] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n02:166745] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n02:166745] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n02:166745] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n02:166745] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n02:166745] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n02:166745] [10] [a26n15:55850] *** Process received signal *** +[a26n15:55850] Signal: Segmentation fault (11) +[a26n15:55850] Signal code: (3) +[a26n15:55850] Failing at address: 0xb170000000000a0 +[a26n15:55850] [ 0] [0x1000000504d8] +[a26n15:55850] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n17:126525] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n17:126525] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n17:126525] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n17:126525] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n17:150932] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n15:161758] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n15:161758] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n15:161758] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n08:127444] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n08:127444] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n08:127444] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n08:127444] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n02:49109] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n02:49109] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n02:49109] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n02:49109] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n11:02686] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n11:02686] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n11:02686] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n11:02686] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n11:02686] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n11:02686] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n11:02686] [10] [a30n14:163551] *** Process received signal *** +[a30n14:163551] Signal: Segmentation fault (11) +[a30n14:163551] Signal code: (3) +[a30n14:163551] Failing at address: 0x14170000000000a0 +[a30n14:163551] [ 0] [0x1000000504d8] +[a30n14:163551] [ 1] [0x3ffffc92cff0] +[a30n14:163551] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n14:163551] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n14:163551] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n10:11493] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n10:11493] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n14:122442] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n14:122442] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n14:122442] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n01:158307] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n01:158307] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n15:151253] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n15:151253] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n15:151253] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n15:151253] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n15:151253] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n10:161718] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n10:161718] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n10:161718] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n10:161718] *** End of error message *** +[a36n09:140256] *** Process received signal *** +[a36n09:140256] Signal: Segmentation fault (11) +[a36n09:140256] Signal code: (3) +[a36n09:140256] Failing at address: 0x5170000000000a0 +[a36n09:140256] [ 0] [0x1000000504d8] +[a36n09:140256] [ 1] [0x3fffca7f5c80] +[a36n09:140256] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n09:140256] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n09:140256] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n13:06928] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n13:06928] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n13:06928] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n09:147792] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n09:147792] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n09:147792] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n09:147792] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n09:147792] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n05:173082] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n05:173082] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n04:13082] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n04:13082] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n04:13082] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n04:13082] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n16:124795] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n16:124795] *** End of error message *** +[b01n08:172428] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n08:172428] [ 6] [a35n11:148673] *** Process received signal *** +[a35n11:148673] Signal: Segmentation fault (11) +[a35n11:148673] Signal code: (3) +[a35n11:148673] Failing at address: 0xb170000000000a0 +[a35n11:148673] [ 0] [0x1000000504d8] +[a35n11:148673] [ 1] [0x3fffc2069d10] +[a35n11:148673] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n11:148673] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n03:125239] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n03:125239] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n03:125239] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n05:162570] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n05:162570] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n05:162570] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n05:162570] [14] [b20n06:130544] *** Process received signal *** +[b20n06:130544] Signal: Segmentation fault (11) +[b20n06:130544] Signal code: (3) +[b20n06:130544] Failing at address: 0x14170000000000a0 +[b20n06:130544] [ 0] [0x1000000504d8] +[b20n06:130544] [ 1] [0x3fffc6d036e0] +[b20n06:130544] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n06:130544] [ 3] [b06n12:177118] *** Process received signal *** +[b06n12:177118] Signal: Segmentation fault (11) +[b06n12:177118] Signal code: (3) +[b06n12:177118] Failing at address: 0xb170000000000a0 +[b06n12:177118] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n17:129934] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n17:129934] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n17:129934] [11] [0x1000000504d8] +[b15n10:122761] [ 1] [0x3fffe3649650] +[b15n10:122761] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n10:122761] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n10:122761] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n05:120229] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n05:120229] *** End of error message *** +[b17n14:126120] [ 0] [0x1000000504d8] +[b17n14:126120] [ 1] [0x3fffc1ed05c0] +[b17n14:126120] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n14:126120] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n14:126120] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n12:160188] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n12:160188] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n12:160188] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n12:160188] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n17:119581] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n17:119581] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n17:119581] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n17:119581] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n17:119581] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n17:119581] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n17:119581] *** End of error message *** +[b05n04:06833] *** Process received signal *** +[b05n04:06833] Signal: Segmentation fault (11) +[b05n04:06833] Signal code: (3) +[b05n04:06833] Failing at address: 0x14170000000000a0 +[b05n04:06833] [ 0] [0x1000000504d8] +[b05n04:06833] [ 1] [0x3fffe250ed80] +[b05n04:06833] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n04:06833] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n04:06833] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n04:06833] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n16:124962] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n16:124962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n16:124962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n16:124962] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n16:124962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n06:38120] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n06:38120] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n06:38120] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n06:38120] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n06:38120] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n06:38120] *** End of error message *** +[a27n05:53376] *** Process received signal *** +[a27n05:53376] Signal: Segmentation fault (11) +[a27n05:53376] Signal code: (3) +[a27n05:53376] Failing at address: 0x14170000000000a0 +[a27n05:53376] [ 0] [0x1000000504d8] +[a27n05:53376] [ 1] [b10n07:116114] *** Process received signal *** +[b10n07:116114] Signal: Segmentation fault (11) +[b10n07:116114] Signal code: (3) +[b10n07:116114] Failing at address: 0x14170000000000a0 +[b10n07:116114] [ 0] [0x1000000504d8] +[b10n07:116114] [ 1] [0x3fffd33c71f0] +[b10n07:116114] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n07:116114] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n02:149229] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n02:149229] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n09:161230] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n09:161230] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n09:161230] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n14:116415] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n14:116415] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n08:118552] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n08:118552] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n08:118552] [13] [0x1000000504d8] +[b02n09:166572] [ 1] [0x3fffe8fc6860] +[b02n09:166572] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n09:166572] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n09:166572] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n09:166572] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n09:166572] [ 6] [b18n18:121303] *** Process received signal *** +[b18n18:121303] Signal: Segmentation fault (11) +[b18n18:121303] Signal code: (3) +[b18n18:121303] Failing at address: 0x14170000000000a0 +[b18n18:121303] [ 0] [0x1000000504d8] +[b18n18:121303] [ 1] [0x3fffc017c7e0] +[b18n18:121303] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n18:121303] [ 3] [b16n12:134494] *** Process received signal *** +[b16n12:134494] Signal: Segmentation fault (11) +[b16n12:134494] Signal code: (3) +[b16n12:134494] Failing at address: 0x14170000000000a0 +[b16n12:134494] [ 0] [0x1000000504d8] +[b16n12:134494] [ 1] [0x3ffff9297520] +[b16n12:134494] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n12:134494] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n12:134494] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n09:152063] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n09:152063] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n09:152063] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n09:152063] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n09:152063] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n16:147288] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n16:147288] [ 8] [b11n07:156144] *** Process received signal *** +[b11n07:156144] Signal: Segmentation fault (11) +[b11n07:156144] Signal code: (3) +[b11n07:156144] Failing at address: 0xb170000000000a0 +[b11n07:156144] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n17:176415] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n17:176415] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n17:176415] *** End of error message *** +[a27n11:23064] *** Process received signal *** +[a27n11:23064] Signal: Segmentation fault (11) +[a27n11:23064] Signal code: (3) +[a27n11:23064] Failing at address: 0x14170000000000a0 +[a27n11:23064] [ 0] [0x1000000504d8] +[a27n11:23064] [ 1] [0x3fffecc839b0] +[a27n11:23064] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n11:23064] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n11:23064] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n11:23064] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n11:23064] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n11:23064] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n11:23064] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n11:23064] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n11:23064] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n11:23064] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n11:23064] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n11:23064] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n11:23064] [14] [a28n12:177495] *** Process received signal *** +[a28n12:177495] Signal: Segmentation fault (11) +[a28n12:177495] Signal code: (3) +[a28n12:177495] Failing at address: 0xb170000000000a0 +[a28n12:177495] [ 0] [0x1000000504d8] +[a28n12:177495] [ 1] [b04n17:08307] *** Process received signal *** +[b04n17:08307] Signal: Segmentation fault (11) +[b04n17:08307] Signal code: (3) +[b04n17:08307] Failing at address: 0xb170000000000a0 +[b04n17:08307] [ 0] [0x1000000504d8] +[b04n17:08307] [ 1] [0x3ffffa215630] +[b04n17:08307] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n04:172231] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n04:172231] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n04:172231] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n03:164496] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n03:164496] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n03:164496] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n18:03315] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n18:03315] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n04:177596] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n04:177596] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n04:177596] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n04:177596] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n17:138617] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n17:138617] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n17:138617] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n17:138617] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n17:138617] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n17:138617] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n17:138617] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n17:138617] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n17:138617] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n16:135793] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n16:135793] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n16:135793] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n10:155401] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n10:155401] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n10:155401] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n10:155401] *** End of error message *** +[a27n17:15872] *** Process received signal *** +[a27n17:15872] Signal: Segmentation fault (11) +[a27n17:15872] Signal code: (3) +[a27n17:15872] Failing at address: 0xb170000000000a0 +[a27n17:15872] [ 0] [0x1000000504d8] +[b18n03:136666] [ 1] [0x3fffe63a1430] +[b18n03:136666] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n03:136666] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n03:136666] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n03:136666] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n03:136666] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n03:136666] [ 7] [b05n16:168050] *** Process received signal *** +[b05n16:168050] Signal: Segmentation fault (11) +[b05n16:168050] Signal code: (3) +[b05n16:168050] Failing at address: 0x5170000000000a0 +[b05n16:168050] [ 0] [0x1000000504d8] +[b05n16:168050] [ 1] [0x3fffda71efe0] +[b05n16:168050] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n16:168050] [ 3] [b20n13:131759] *** Process received signal *** +[b20n13:131759] Signal: Segmentation fault (11) +[b20n13:131759] Signal code: (3) +[b20n13:131759] Failing at address: 0xb170000000000a0 +[b20n13:131759] [ 0] [0x1000000504d8] +[b20n13:131759] [ 1] [0x3fffed7fb590] +[b20n13:131759] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n13:131759] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n13:131759] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n13:131759] [ 5] [b23n04:117354] *** Process received signal *** +[b23n04:117354] Signal: Segmentation fault (11) +[b23n04:117354] Signal code: (3) +[b23n04:117354] Failing at address: 0x14170000000000a0 +[b23n04:117354] [ 0] [0x1000000504d8] +[b23n04:117354] [ 1] [0x3fffefc11cd0] +[b23n04:117354] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n01:138008] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n01:138008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n01:138008] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n01:138008] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n14:165661] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n14:165661] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n14:165661] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n09:143901] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n09:143901] [ 7] [b12n13:130059] *** Process received signal *** +[b12n13:130059] Signal: Segmentation fault (11) +[b12n13:130059] Signal code: (3) +[b12n13:130059] Failing at address: 0x5170000000000a0 +[b12n13:130059] [ 0] [0x1000000504d8] +[b12n13:130059] [ 1] [0x3fffe4af19b0] +[b12n13:130059] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n09:62481] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n09:62481] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n09:62481] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n09:62481] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n09:62481] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n09:62481] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n09:62481] [10] [0x1000000504d8] +[b12n14:139845] [ 1] [0x3fffd351d570] +[b12n14:139845] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n14:139845] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n14:139845] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n10:169682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n10:169682] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n13:123518] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n13:123518] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n13:123518] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n13:123518] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n17:168164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n17:168164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n17:168164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n17:168164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n17:168164] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n17:168164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n17:168164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n17:168164] [13] [b20n14:120638] *** Process received signal *** +[b20n14:120638] Signal: Segmentation fault (11) +[b20n14:120638] Signal code: (3) +[b20n14:120638] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n07:172881] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n07:172881] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n10:126192] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n10:126192] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n10:126192] [ 8] [b16n13:122188] *** Process received signal *** +[b16n13:122188] Signal: Segmentation fault (11) +[b16n13:122188] Signal code: (3) +[b16n13:122188] Failing at address: 0xb170000000000a0 +[b16n13:122188] [ 0] [0x1000000504d8] +[b16n13:122188] [ 1] [0x3ffffb526260] +[b16n13:122188] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n13:122188] [ 3] [a28n15:07537] *** Process received signal *** +[a28n15:07537] Signal: Segmentation fault (11) +[a28n15:07537] Signal code: (3) +[a28n15:07537] Failing at address: 0x14170000000000a0 +[a28n15:07537] [ 0] [0x1000000504d8] +[a28n15:07537] [ 1] [0x3ffffb200ab0] +[a28n15:07537] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n15:07537] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n15:07537] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n15:07537] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n15:07537] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n15:07537] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n15:07537] [ 8] [b13n08:144853] *** Process received signal *** +[b13n08:144853] Signal: Segmentation fault (11) +[b13n08:144853] Signal code: (3) +[b13n08:144853] Failing at address: 0xb170000000000a0 +[b13n08:144853] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n13:177641] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n13:177641] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n13:177641] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n13:177641] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n13:177641] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n13:177641] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n07:131674] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n07:131674] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n07:131674] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n07:131674] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n07:131674] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n07:131674] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n07:131674] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n07:131674] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n07:131674] [12] [b17n12:123235] *** End of error message *** +[b02n12:180116] *** Process received signal *** +[b02n12:180116] Signal: Segmentation fault (11) +[b02n12:180116] Signal code: (3) +[b02n12:180116] Failing at address: 0x14170000000000a0 +[b02n12:180116] [ 0] [0x1000000504d8] +[b02n12:180116] [ 1] [0x3ffff8c583c0] +[b02n12:180116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n12:180116] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n12:164381] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n12:164381] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n12:164381] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n18:164801] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n18:164801] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n18:164801] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n18:164801] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n13:138089] [ 6] [b18n12:120402] *** Process received signal *** +[b18n12:120402] Signal: Segmentation fault (11) +[b18n12:120402] Signal code: (3) +[b18n12:120402] Failing at address: 0xb170000000000a0 +[b18n12:120402] [ 0] [0x1000000504d8] +[b18n12:120402] [ 1] [0x3fffcdf68b80] +[b18n12:120402] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n12:120402] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n12:120402] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n12:120402] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n12:120402] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x464)[0x1000031d90d4] +[b18n12:120402] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n12:120402] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n12:120402] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n12:120402] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n12:120402] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n13:163196] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n13:163196] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n13:155852] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n13:155852] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n13:155852] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n13:133628] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n13:133628] [13] [b09n07:173042] *** Process received signal *** +[b09n07:173042] Signal: Segmentation fault (11) +[b09n07:173042] Signal code: (3) +[b09n07:173042] Failing at address: 0xb170000000000a0 +[b09n07:173042] [ 0] [0x1000000504d8] +[b09n07:173042] [ 1] [0x3fffe16d03a0] +[b09n07:173042] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n07:173042] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n15:32402] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n15:32402] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n15:32402] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n15:32402] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n15:32402] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n15:32402] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n08:141875] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n08:141875] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n08:141875] [13] [0x1000000504d8] +[b18n08:131378] [ 1] [0x3fffd1409180] +[b18n08:131378] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n08:131378] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n08:131378] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n06:164016] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n06:164016] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n06:164016] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n06:164016] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n18:176812] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n18:176812] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n18:176812] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n14:08400] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n14:08400] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n14:08400] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n14:08400] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n17:118573] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n17:118573] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n17:118573] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n17:118573] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n02:167597] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n02:167597] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n18:173977] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n18:173977] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n18:173977] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n18:173977] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n14:119485] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n14:119485] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n14:119485] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n14:119485] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n14:119485] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n14:119485] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n14:119485] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n11:162471] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n11:162471] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n11:162471] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n08:118580] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n15:119251] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n15:119251] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n14:150132] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n14:150132] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n14:150132] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n01:128003] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n03:177924] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n03:177924] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n03:177924] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n03:177924] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n10:118388] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n10:118388] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n10:118388] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n10:118388] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n10:126790] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n15:154384] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n15:154384] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n15:154384] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n15:154384] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n14:173894] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n14:173894] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n14:173894] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n14:173894] [ 8] [b21n13:117511] *** Process received signal *** +[b21n13:117511] Signal: Segmentation fault (11) +[b21n13:117511] Signal code: (3) +[b21n13:117511] Failing at address: 0xb170000000000a0 +[b21n13:117511] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n09:123417] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n09:123417] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n09:123417] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n09:123417] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n02:155384] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n02:155384] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n02:155384] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n02:155384] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n14:136660] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n14:136660] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n14:136660] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n10:134735] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n10:134735] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n10:134735] [ 6] [b02n17:03845] *** Process received signal *** +[b02n17:03845] Signal: Segmentation fault (11) +[b02n17:03845] Signal code: (3) +[b02n17:03845] Failing at address: 0xb170000000000a0 +[b02n17:03845] [ 0] [0x1000000504d8] +[b02n17:03845] [ 1] [0x3fffca3c59c0] +[b02n17:03845] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n17:03845] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n17:03845] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n09:123815] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n09:123815] [10] [b09n16:173549] *** Process received signal *** +[b09n16:173549] Signal: Segmentation fault (11) +[b09n16:173549] Signal code: (3) +[b09n16:173549] Failing at address: 0xb170000000000a0 +[b09n16:173549] [ 0] [0x1000000504d8] +[b09n16:173549] [ 1] [0x3fffd08503c0] +[b09n16:173549] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n16:173549] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n18:08432] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n18:08432] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n11:160797] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n11:160797] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n11:160797] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n11:163262] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n11:163262] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n11:163262] [ 6] [0x3fffe4e48200] +[a32n05:154190] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n05:154190] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n05:154190] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n05:154190] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n05:154190] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n04:133902] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n04:133902] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n04:133902] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n04:133902] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n03:158660] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n03:158660] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n14:21318] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n14:21318] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n14:21318] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n06:135850] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n06:135850] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n06:135850] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n18:177222] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n18:177222] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n18:177222] [ 6] [b21n12:120130] *** Process received signal *** +[b21n12:120130] Signal: Segmentation fault (11) +[b21n12:120130] Signal code: (3) +[b21n12:120130] Failing at address: 0x14170000000000a0 +[b21n12:120130] [ 0] [0x1000000504d8] +[b21n12:120130] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n12:148380] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n12:148380] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n12:148380] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n12:148380] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n12:148380] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n18:140385] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n18:140385] *** End of error message *** +[a31n11:153420] *** Process received signal *** +[a31n11:153420] Signal: Segmentation fault (11) +[a31n11:153420] Signal code: (3) +[a31n11:153420] Failing at address: 0x14170000000000a0 +[a31n11:153420] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n16:149064] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n16:149064] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n16:149064] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n07:133377] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n07:133377] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n07:133377] [ 5] [b01n10:173683] *** Process received signal *** +[b01n10:173683] Signal: Segmentation fault (11) +[b01n10:173683] Signal code: (3) +[b01n10:173683] Failing at address: 0x14170000000000a0 +[b01n10:173683] [ 0] [0x1000000504d8] +[b01n10:173683] [ 1] [0x3fffe3baa510] +[b01n10:173683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n10:173683] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n05:138196] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n05:138196] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n07:125449] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n07:125449] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n07:125449] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n07:125449] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n07:125449] [11] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n18:138475] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n18:138475] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n18:138475] [ 5] [b07n16:166499] *** Process received signal *** +[b07n16:166499] Signal: Segmentation fault (11) +[b07n16:166499] Signal code: (3) +[b07n16:166499] Failing at address: 0x14170000000000a0 +[b07n16:166499] [ 0] [0x1000000504d8] +[b07n16:166499] [ 1] [0x3fffd9a9d2a0] +[b07n16:166499] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n16:166499] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n18:07425] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n18:07425] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n18:07425] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n18:07425] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n18:07425] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n09:168246] [13] [a26n05:54536] *** Process received signal *** +[a26n05:54536] Signal: Segmentation fault (11) +[a26n05:54536] Signal code: (3) +[a26n05:54536] Failing at address: 0xb170000000000a0 +[a26n05:54536] [ 0] [0x1000000504d8] +[a26n05:54536] [ 1] [0x3fffeb26eed0] +[a26n05:54536] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n05:54536] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n05:54536] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n03:119832] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n03:119832] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n03:119832] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n03:119832] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n17:171201] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n17:171201] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n17:171201] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n17:171201] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n17:171201] [ 8] [b21n08:116171] *** Process received signal *** +[b21n08:116171] Signal: Segmentation fault (11) +[b21n08:116171] Signal code: (3) +[b21n08:116171] Failing at address: 0xb170000000000a0 +[b21n08:116171] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n16:165921] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n16:165921] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n16:165921] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n16:165921] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n16:165921] [14] [a27n04:52704] *** Process received signal *** +[a27n04:52704] Signal: Segmentation fault (11) +[a27n04:52704] Signal code: (3) +[a27n04:52704] Failing at address: 0x14170000000000a0 +[a27n04:52704] [ 0] [0x1000000504d8] +[a27n04:52704] [ 1] [0x3fffd3bccb20] +[a27n04:52704] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n04:52704] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n04:52704] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n04:52704] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n04:52704] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n04:52704] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n04:52704] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n04:52704] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n04:52704] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n04:52704] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n04:52704] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n04:52704] [13] [b21n07:117359] Signal: Segmentation fault (11) +[b21n07:117359] Signal code: (3) +[b21n07:117359] Failing at address: 0xb170000000000a0 +[b21n07:117359] [ 0] [0x1000000504d8] +[b21n07:117359] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n16:127354] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n16:127354] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n16:127354] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n13:122034] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n07:124350] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n07:124350] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n07:124350] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n07:124350] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n17:124165] *** End of error message *** +[a02n09:160374] *** Process received signal *** +[a02n09:160374] Signal: Segmentation fault (11) +[a02n09:160374] Signal code: (3) +[a02n09:160374] Failing at address: 0x14170000000000a0 +[a26n18:69156] *** Process received signal *** +[a26n18:69156] Signal: Segmentation fault (11) +[a26n18:69156] Signal code: (3) +[a26n18:69156] Failing at address: 0x14170000000000a0 +[a26n18:69156] [ 0] [b16n11:124434] *** Process received signal *** +[b16n11:124434] Signal: Segmentation fault (11) +[b16n11:124434] Signal code: (3) +[b16n11:124434] Failing at address: 0xb170000000000a0 +[b16n11:124434] [ 0] [0x1000000504d8] +[b16n11:124434] [ 1] [b13n16:128443] *** Process received signal *** +[b13n16:128443] Signal: Segmentation fault (11) +[b13n16:128443] Signal code: (3) +[b13n16:128443] Failing at address: 0x14170000000000a0 +[b13n16:128443] [ 0] [0x1000000504d8] +[b13n16:128443] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n10:124501] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n10:124501] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n10:124501] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n14:177223] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n14:177223] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n14:177223] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n14:177223] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n14:177223] *** End of error message *** +[0x1000000504d8] +[a02n03:38501] [ 1] [0x3fffee54be10] +[a02n03:38501] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n03:38501] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n01:48804] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n01:48804] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n01:48804] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n01:48804] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n01:48804] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n01:48804] *** End of error message *** +[ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n14:08951] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n14:08951] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n14:08951] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n12:12766] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n12:12766] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n12:167677] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n12:167677] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n12:167677] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n12:167677] [ 9] [a29n15:177059] *** Process received signal *** +[a29n15:177059] Signal: Segmentation fault (11) +[a29n15:177059] Signal code: (3) +[a29n15:177059] Failing at address: 0xb170000000000a0 +[a29n15:177059] [ 0] [0x1000000504d8] +[a29n15:177059] [ 1] [b23n13:114397] *** Process received signal *** +[b23n13:114397] Signal: Segmentation fault (11) +[b23n13:114397] Signal code: (3) +[b23n13:114397] Failing at address: 0x14170000000000a0 +[b23n13:114397] [ 0] [0x1000000504d8] +[b23n13:114397] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n14:152746] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n14:152746] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n03:147538] *** End of error message *** +[a36n10:142929] *** Process received signal *** +[a36n10:142929] Signal: Segmentation fault (11) +[a36n10:142929] Signal code: (3) +[a36n10:142929] Failing at address: 0xb170000000000a0 +[a36n10:142929] [ 0] [0x1000000504d8] +[a36n10:142929] [b11n14:161616] *** Process received signal *** +[b11n14:161616] Signal: Segmentation fault (11) +[b11n14:161616] Signal code: (3) +[b11n14:161616] Failing at address: 0xb170000000000a0 +[b11n14:161616] [ 0] [b21n16:116525] *** Process received signal *** +[b21n16:116525] Signal: Segmentation fault (11) +[b21n16:116525] Signal code: (3) +[b21n16:116525] Failing at address: 0x14170000000000a0 +[b21n16:116525] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n11:160351] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n11:160351] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n11:160351] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n11:160351] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n11:160351] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n01:145732] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n17:149905] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n17:149905] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n17:149905] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n17:149905] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n05:135164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n05:135164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n05:135164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n05:135164] [10] [a29n16:157888] *** Process received signal *** +[a29n16:157888] Signal: Segmentation fault (11) +[a29n16:157888] Signal code: (3) +[a29n16:157888] Failing at address: 0xb170000000000a0 +[a29n16:157888] [ 0] [0x1000000504d8] +[a29n16:157888] [ 1] [0x3fffe90eea20] +[a29n16:157888] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n13:165389] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n13:165389] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n13:165389] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n13:165389] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n13:165389] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n07:118168] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n07:118168] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n07:118168] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n07:118168] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n07:118168] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n14:128869] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n14:128869] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n09:03928] [ 5] [a28n10:176311] *** Process received signal *** +[a28n10:176311] Signal: Segmentation fault (11) +[a28n10:176311] Signal code: (3) +[a28n10:176311] Failing at address: 0x14170000000000a0 +[a28n10:176311] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n17:119466] *** End of error message *** +[a26n11:62421] *** Process received signal *** +[a26n11:62421] Signal: Segmentation fault (11) +[a26n11:62421] Signal code: (3) +[a26n11:62421] Failing at address: 0xb170000000000a0 +[a26n11:62421] [ 0] [0x1000000504d8] +[a26n11:62421] [ 1] [0x3fffc800ece0] +[a26n11:62421] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n14:133633] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n14:133633] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n14:133633] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n14:133633] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n14:169523] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n14:169523] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n14:169523] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n14:169523] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n14:169523] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n14:169523] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n14:169523] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n14:169523] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n14:169523] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n10:10927] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n10:10927] *** End of error message *** +[0x1000000504d8] +[b14n06:127277] [ 1] [0x3fffcf095170] +[b14n06:127277] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n06:127277] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n06:127277] [ 4] [b09n11:171963] *** Process received signal *** +[b09n11:171963] Signal: Segmentation fault (11) +[b09n11:171963] Signal code: (3) +[b09n11:171963] Failing at address: 0xb170000000000a0 +[b09n11:171963] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n18:179701] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n18:179701] *** End of error message *** +[a33n01:141816] *** Process received signal *** +[a33n01:141816] Signal: Segmentation fault (11) +[a33n01:141816] Signal code: (3) +[a33n01:141816] Failing at address: 0xb170000000000a0 +[a33n01:141816] [ 0] [0x1000000504d8] +[a33n01:141816] [ 1] [0x3fffd80df590] +[a33n01:141816] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n01:141816] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n01:141816] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n01:179261] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n01:179261] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n02:119535] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n02:119535] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n02:119535] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n16:53795] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n16:53795] *** End of error message *** +[b20n16:124664] *** Process received signal *** +[b20n16:124664] Signal: Segmentation fault (11) +[b20n16:124664] Signal code: (3) +[b20n16:124664] Failing at address: 0xb170000000000a0 +[b20n16:124664] [ 0] [0x1000000504d8] +[b20n16:124664] [ 1] [0x3fffd71d99b0] +[b20n16:124664] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n16:124664] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n11:128958] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n11:128958] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n11:128958] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n11:128958] [14] [a36n06:143012] *** Process received signal *** +[a36n06:143012] Signal: Segmentation fault (11) +[a36n06:143012] Signal code: (3) +[a36n06:143012] Failing at address: 0x5170000000000a0 +[a36n06:143012] [ 0] [0x1000000504d8] +[a36n06:143012] [ 1] [0x3ffff1a4da20] +[a36n06:143012] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n06:143012] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n07:153176] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n07:153176] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n07:153176] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n07:153176] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n07:153176] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n04:176001] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n04:176001] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n04:176001] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n04:176001] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n04:176001] *** End of error message *** +[a36n13:156921] *** Process received signal *** +[a36n13:156921] Signal: Segmentation fault (11) +[a36n13:156921] Signal code: (3) +[a36n13:156921] Failing at address: 0xb170000000000a0 +[a36n13:156921] [ 0] [0x1000000504d8] +[a36n13:156921] [ 1] [0x3fffca9b5be0] +[a36n13:156921] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n13:156921] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n08:147821] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n08:147821] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n08:147821] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n08:147821] [13] [0x1000000504d8] +[a33n04:169911] [ 1] [0x3ffff2eef250] +[a33n04:169911] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n04:169911] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n04:169911] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n18:28218] *** End of error message *** +[b05n08:08957] *** Process received signal *** +[b05n08:08957] Signal: Segmentation fault (11) +[b05n08:08957] Signal code: (3) +[b05n08:08957] Failing at address: 0x5170000000000a0 +[b05n08:08957] [ 0] [0x1000000504d8] +[b05n08:08957] [ 1] [0x3fffd5d20720] +[b05n08:08957] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n08:08957] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n02:164493] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n02:164493] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n13:176368] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n13:176368] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n13:176368] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n13:176368] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n13:176368] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n13:176368] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n13:176368] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n13:176368] [13] [b11n11:152987] *** Process received signal *** +[b11n11:152987] Signal: Segmentation fault (11) +[b11n11:152987] Signal code: (3) +[b11n11:152987] Failing at address: 0xb170000000000a0 +[b11n11:152987] [ 0] [0x1000000504d8] +[b11n11:152987] [ 1] [0x3fffcb0fb090] +[b11n11:152987] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n11:152987] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n06:148312] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n06:148312] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n13:123707] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n13:123707] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n04:126859] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n04:126859] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n04:126859] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n18:126908] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n18:126908] [ 6] [b20n07:132089] *** Process received signal *** +[b20n07:132089] Signal: Segmentation fault (11) +[b20n07:132089] Signal code: (3) +[b20n07:132089] Failing at address: 0x14170000000000a0 +[b20n07:132089] [ 0] [0x1000000504d8] +[b20n07:132089] [ 1] [0x3fffed03bf50] +[b20n07:132089] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n15:85940] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n15:85940] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n15:85940] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n15:85940] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n09:120696] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n09:120696] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n09:120696] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n09:120696] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n09:120696] *** End of error message *** +[b02n16:01573] *** Process received signal *** +[b02n16:01573] Signal: Segmentation fault (11) +[b02n16:01573] Signal code: (3) +[b02n16:01573] Failing at address: 0xb170000000000a0 +[b02n16:01573] [ 0] [0x1000000504d8] +[b02n16:01573] [ 1] [0x3fffc67f0370] +[b02n16:01573] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n16:01573] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n09:173750] [ 5] [b21n10:130583] Signal: Segmentation fault (11) +[b21n10:130583] Signal code: (3) +[b21n10:130583] Failing at address: 0x14170000000000a0 +[b21n10:130583] [ 0] [0x1000000504d8] +[b21n10:130583] [ 1] [0x3fffffa08d60] +[b21n10:130583] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n10:130583] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n03:10290] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n03:10290] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n03:10290] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n03:10290] *** End of error message *** +[a28n11:01170] *** Process received signal *** +[a28n11:01170] Signal: Segmentation fault (11) +[a28n11:01170] Signal code: (3) +[a28n11:01170] Failing at address: 0xb170000000000a0 +[a28n11:01170] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n16:173909] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n14:168376] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n14:168376] [ 5] [0x3fffe0953af0] +[b20n08:124725] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n08:124725] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n08:124725] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n09:119547] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n09:119547] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n09:119547] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n09:119547] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n13:01301] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n13:01301] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n13:01301] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n13:01301] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n13:01301] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n18:164009] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n16:153290] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n16:153290] [ 5] [b08n16:171918] *** Process received signal *** +[b08n16:171918] Signal: Segmentation fault (11) +[b08n16:171918] Signal code: (3) +[b08n16:171918] Failing at address: 0x14170000000000a0 +[b08n16:171918] [ 0] [0x1000000504d8] +[b08n16:171918] [ 1] [0x3fffcb1329f0] +[b08n16:171918] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n16:171918] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n15:177862] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n15:177862] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n15:177862] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n15:177862] [ 9] [b12n11:137871] *** End of error message *** +[a26n15:55851] *** Process received signal *** +[a26n15:55851] Signal: Segmentation fault (11) +[a26n15:55851] Signal code: (3) +[a26n15:55851] Failing at address: 0xb170000000000a0 +[a26n15:55851] [ 0] [0x1000000504d8] +[a26n15:55851] [ 1] [0x3ffff8030fc0] +[a26n15:55851] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n02:166748] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n02:166748] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n02:166748] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n02:166748] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n02:166748] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n17:150932] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n17:126525] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n17:126525] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n15:161755] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n15:161755] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n15:161755] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n15:161755] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n02:49109] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n11:02685] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n11:02685] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n11:02685] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n11:02685] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n11:02685] [11] [a30n14:163548] *** Process received signal *** +[a30n14:163548] Signal: Segmentation fault (11) +[a30n14:163548] Signal code: (3) +[a30n14:163548] Failing at address: 0xb170000000000a0 +[a30n14:163548] [ 0] [0x1000000504d8] +[a30n14:163548] [ 1] [0x3ffff246e930] +[a30n14:163548] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n14:163548] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n14:163548] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n08:127444] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n08:127444] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n08:127444] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n15:151253] *** End of error message *** +[b06n17:163815] *** Process received signal *** +[b06n17:163815] Signal: Segmentation fault (11) +[b06n17:163815] Signal code: (3) +[b06n17:163815] Failing at address: 0x14170000000000a0 +[b06n17:163815] [ 0] [0x1000000504d8] +[b06n17:163815] [ 1] [0x3fffcad36bf0] +[b06n17:163815] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n17:163815] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n17:163815] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n17:163815] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n17:163815] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n17:163815] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n17:163815] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n17:163815] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n17:163815] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n17:163815] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n17:163815] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n17:163815] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n17:163815] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n17:163815] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n13:06927] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n13:06927] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n13:06927] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n13:06927] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n04:13083] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n09:147795] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n09:147795] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n09:147795] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n16:124796] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n16:124796] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n16:124796] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n16:124796] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n16:124796] *** End of error message *** +[a30n01:163389] *** Process received signal *** +[a30n01:163389] Signal: Segmentation fault (11) +[a30n01:163389] Signal code: (3) +[a30n01:163389] Failing at address: 0xb170000000000a0 +[a30n01:163389] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n03:125239] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n03:125239] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n03:125239] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n03:125239] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n05:162570] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n11:148673] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n01:164962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n01:164962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n01:164962] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n01:164962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n01:164962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n08:172428] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n08:172428] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n08:172428] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n08:172428] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n08:172428] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n08:172428] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n08:172428] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n18:10725] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n18:10725] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n18:10725] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n18:10725] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n18:10725] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n18:10725] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n06:130544] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n06:130544] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n06:130544] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n11:164998] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n11:164998] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n11:164998] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n11:164998] *** End of error message *** +[b15n02:122560] *** Process received signal *** +[b15n02:122560] Signal: Segmentation fault (11) +[b15n02:122560] Signal code: (3) +[b15n02:122560] Failing at address: 0x14170000000000a0 +[b15n02:122560] [ 0] [0x1000000504d8] +[b15n02:122560] [ 1] [0x3fffecac9fc0] +[b15n02:122560] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n02:122560] [ 3] [a32n03:151881] *** Process received signal *** +[a32n03:151881] Signal: Segmentation fault (11) +[a32n03:151881] Signal code: (3) +[a32n03:151881] Failing at address: 0xb170000000000a0 +[a32n03:151881] [ 0] [0x1000000504d8] +[a32n03:151881] [ 1] [0x3fffc4aff280] +[a32n03:151881] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n03:151881] [ 3] [b11n10:161720] *** Process received signal *** +[b11n10:161720] Signal: Segmentation fault (11) +[b11n10:161720] Signal code: (3) +[b11n10:161720] Failing at address: 0xb170000000000a0 +[b11n10:161720] [ 0] [0x1000000504d8] +[b11n10:161720] [ 1] [0x3fffea335170] +[b11n10:161720] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n10:161720] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n15:155431] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n15:155431] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n15:155431] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n15:155431] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n15:155431] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n15:155431] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n15:155431] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n09:155761] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n09:155761] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n09:155761] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n09:155761] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n09:155761] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n09:155761] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n09:155761] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n09:155761] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n09:155761] *** End of error message *** +[b10n06:162919] *** Process received signal *** +[b10n06:162919] Signal: Segmentation fault (11) +[b10n06:162919] Signal code: (3) +[b10n06:162919] Failing at address: 0xb170000000000a0 +[b10n06:162919] [ 0] [0x1000000504d8] +[b10n06:162919] [ 1] [0x3fffed5360e0] +[b10n06:162919] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n06:162919] [ 3] [a29n10:164939] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n05:173848] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n05:173848] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n06:133957] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n06:133957] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n06:133957] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n06:133957] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n17:164564] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n17:164564] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n17:164564] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n17:164564] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n17:164564] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n17:164564] *** End of error message *** +[b08n01:171788] *** Process received signal *** +[b08n01:171788] Signal: Segmentation fault (11) +[b08n01:171788] Signal code: (3) +[b08n01:171788] Failing at address: 0x14170000000000a0 +[b08n01:171788] [ 0] [0x1000000504d8] +[b08n01:171788] [ 1] [0x3fffe91e1130] +[b08n01:171788] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n01:171788] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n01:171788] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n01:171788] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n01:171788] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n01:171788] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n01:171788] [ 8] [0x3ffffaf1d310] +[b15n05:120331] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n05:120331] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n05:120331] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n05:120331] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n08:129890] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n08:129890] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n14:126466] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n14:126466] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n14:126466] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n14:126466] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n08:164750] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n08:164750] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n12:165845] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n12:165845] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n12:165845] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n12:165845] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n12:165845] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n12:165845] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n12:165845] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n12:165845] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n12:165845] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n04:130746] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n04:130746] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n04:130746] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n04:130746] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n12:156916] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n12:156916] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n12:156916] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n12:156916] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n15:177821] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n15:177821] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n16:164551] [ 4] [0x1000000504d8] +[a31n02:158418] [ 1] [0x3fffe83f6070] +[a31n02:158418] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n02:158418] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n02:158418] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n02:158418] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n02:158418] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n03:175064] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n03:175064] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n03:175064] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n03:175064] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n03:175064] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n03:175064] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n03:175064] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n03:175064] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n03:175064] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n03:175064] [14] [b15n01:123571] *** Process received signal *** +[b15n01:123571] Signal: Segmentation fault (11) +[b15n01:123571] Signal code: (3) +[b15n01:123571] Failing at address: 0x14170000000000a0 +[b15n01:123571] [ 0] [0x1000000504d8] +[b15n01:123571] [ 1] [0x3fffc2ec5ff0] +[b15n01:123571] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n01:123571] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n05:173081] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n05:173081] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n05:173081] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n05:173081] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n05:173081] [ 8] [a26n07:60173] *** Process received signal *** +[a26n07:60173] Signal: Segmentation fault (11) +[a26n07:60173] Signal code: (3) +[a26n07:60173] Failing at address: 0x14170000000000a0 +[a26n07:60173] [ 0] [0x1000000504d8] +[a26n07:60173] [ 1] [0x3fffef1c72a0] +[a26n07:60173] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n07:60173] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n13:158867] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n13:158867] *** End of error message *** +[b12n12:138737] *** Process received signal *** +[b12n12:138737] Signal: Segmentation fault (11) +[b12n12:138737] Signal code: (3) +[b12n12:138737] Failing at address: 0x5170000000000a0 +[b12n12:138737] [ 0] [a29n17:169737] *** Process received signal *** +[a29n17:169737] Signal: Segmentation fault (11) +[a29n17:169737] Signal code: (3) +[a29n17:169737] Failing at address: 0x14170000000000a0 +[a29n17:169737] [ 0] [0x1000000504d8] +[a29n17:169737] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n06:167372] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n09:174690] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n09:174690] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n09:174690] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n09:174690] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n09:174690] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n09:174690] [ 9] [a33n17:168156] *** Process received signal *** +[a33n17:168156] Signal: Segmentation fault (11) +[a33n17:168156] Signal code: (3) +[a33n17:168156] Failing at address: 0xb170000000000a0 +[a33n17:168156] [ 0] [0x1000000504d8] +[a33n17:168156] [ 1] [0x3ffff801af80] +[a33n17:168156] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n17:168156] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n17:168156] [ 4] [b23n06:117525] *** Process received signal *** +[b23n06:117525] Signal: Segmentation fault (11) +[b23n06:117525] Signal code: (3) +[b23n06:117525] Failing at address: 0xb170000000000a0 +[b23n06:117525] [ 0] [0x1000000504d8] +[b23n06:117525] [ 1] [0x3fffcd192fb0] +[b23n06:117525] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n06:117525] [ 3] [a02n07:152085] *** Process received signal *** +[a02n07:152085] Signal: Segmentation fault (11) +[a02n07:152085] Signal code: (3) +[a02n07:152085] Failing at address: 0xb170000000000a0 +[a02n07:152085] [ 0] [0x1000000504d8] +[a02n07:152085] [ 1] [0x3fffd84d90d0] +[a02n07:152085] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n07:152085] [ 3] [b10n01:165125] *** Process received signal *** +[b10n01:165125] Signal: Segmentation fault (11) +[b10n01:165125] Signal code: (3) +[b10n01:165125] Failing at address: 0x14170000000000a0 +[b10n01:165125] [ 0] [0x1000000504d8] +[b10n01:165125] [ 1] [0x3fffe45e1390] +[b10n01:165125] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n01:165125] [ 3] [a36n12:141259] *** Process received signal *** +[a36n12:141259] Signal: Segmentation fault (11) +[a36n12:141259] Signal code: (3) +[a36n12:141259] Failing at address: 0x14170000000000a0 +[a36n12:141259] [ 0] [0x1000000504d8] +[a36n12:141259] [ 1] [0x3ffff6ffcd60] +[a36n12:141259] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n12:141259] [ 3] [b11n03:154295] *** Process received signal *** +[b11n03:154295] Signal: Segmentation fault (11) +[b11n03:154295] Signal code: (3) +[b11n03:154295] Failing at address: 0x14170000000000a0 +[b11n03:154295] [ 0] [0x1000000504d8] +[b11n03:154295] [ 1] [0x3ffff1720fe0] +[b11n03:154295] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n03:154295] [ 3] [b08n06:161720] *** Process received signal *** +[b08n06:161720] Signal: Segmentation fault (11) +[b08n06:161720] Signal code: (3) +[b08n06:161720] Failing at address: 0xb170000000000a0 +[b08n06:161720] [ 0] [0x1000000504d8] +[b08n06:161720] [ 1] [0x3fffea027c50] +[b08n06:161720] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n06:161720] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n06:161720] [ 4] [a35n06:138764] *** Process received signal *** +[a35n06:138764] Signal: Segmentation fault (11) +[a35n06:138764] Signal code: (3) +[a35n06:138764] Failing at address: 0xb170000000000a0 +[a35n06:138764] [ 0] [0x1000000504d8] +[a35n06:138764] [ 1] [0x3fffc368ba80] +[a35n06:138764] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n06:138764] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n06:138764] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n06:138764] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n06:138764] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n06:138764] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n06:138764] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n06:138764] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n06:138764] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n06:138764] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n06:138764] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n06:138764] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n06:138764] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n06:138764] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n14:178324] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n14:178324] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n14:178324] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n14:178324] [13] [b01n12:178596] *** Process received signal *** +[b01n12:178596] Signal: Segmentation fault (11) +[b01n12:178596] Signal code: (3) +[b01n12:178596] Failing at address: 0xb170000000000a0 +[b01n12:178596] [ 0] [0x1000000504d8] +[b01n12:178596] [ 1] [0x3fffc4543b40] +[b01n12:178596] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n12:178596] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n16:162630] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n16:162630] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n16:162630] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n16:162630] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n16:162630] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n16:162630] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n16:162630] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n16:162630] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n16:162630] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n16:162630] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n16:162630] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n16:162630] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n16:162630] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n18:118685] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n18:118685] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n08:156370] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n07:156491] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n07:156491] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n07:156491] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n07:156491] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n01:122016] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n01:122016] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n01:122016] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n01:122016] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n01:122016] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n01:122016] [10] [b23n11:117475] *** Process received signal *** +[b23n11:117475] Signal: Segmentation fault (11) +[b23n11:117475] Signal code: (3) +[b23n11:117475] Failing at address: 0x14170000000000a0 +[b23n11:117475] [ 0] [0x1000000504d8] +[b23n11:117475] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n04:158633] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n04:158633] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n04:158633] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n04:158633] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n04:158633] [ 8] [b08n07:168963] *** Process received signal *** +[b08n07:168963] Signal: Segmentation fault (11) +[b08n07:168963] Signal code: (3) +[b08n07:168963] Failing at address: 0x14170000000000a0 +[b08n07:168963] [ 0] [0x1000000504d8] +[b08n07:168963] [ 1] [a35n04:145750] *** Process received signal *** +[a35n04:145750] Signal: Segmentation fault (11) +[a35n04:145750] Signal code: (3) +[a35n04:145750] Failing at address: 0x14170000000000a0 +[a35n04:145750] [ 0] [0x1000000504d8] +[a35n04:145750] [ 1] [a29n06:157791] *** Process received signal *** +[a29n06:157791] Signal: Segmentation fault (11) +[a29n06:157791] Signal code: (3) +[a29n06:157791] Failing at address: 0x14170000000000a0 +[a29n06:157791] [ 0] [0x1000000504d8] +[a29n06:157791] [ 1] [0x3fffc6cff940] +[a29n06:157791] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n10:11490] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n10:11490] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n10:11490] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n10:11490] *** End of error message *** +[b02n13:171109] *** Process received signal *** +[b02n13:171109] Signal: Segmentation fault (11) +[b02n13:171109] Signal code: (3) +[b02n13:171109] Failing at address: 0x14170000000000a0 +[b02n13:171109] [ 0] [b03n17:176827] *** Process received signal *** +[b03n17:176827] Signal: Segmentation fault (11) +[b03n17:176827] Signal code: (3) +[b03n17:176827] Failing at address: 0xb170000000000a0 +[b03n17:176827] [ 0] [0x1000000504d8] +[b03n17:176827] [ 1] [0x3ffffa2cdf90] +[b03n17:176827] [ 2] [b21n11:120580] *** Process received signal *** +[b21n11:120580] Signal: Segmentation fault (11) +[b21n11:120580] Signal code: (3) +[b21n11:120580] Failing at address: 0xb170000000000a0 +[b21n11:120580] [ 0] [b23n09:119634] *** Process received signal *** +[b23n09:119634] Signal: Segmentation fault (11) +[b23n09:119634] Signal code: (3) +[b23n09:119634] Failing at address: 0x14170000000000a0 +[b23n09:119634] [ 0] [0x1000000504d8] +[b23n09:119634] [ 1] [a27n03:46688] *** Process received signal *** +[a27n03:46688] Signal: Segmentation fault (11) +[a27n03:46688] Signal code: (3) +[a27n03:46688] Failing at address: 0x14170000000000a0 +[a27n03:46688] [ 0] [0x1000000504d8] +[a27n03:46688] [ 1] [0x3fffd9081b40] +[a27n03:46688] [ 2] [a29n07:160802] *** Process received signal *** +[b05n17:177403] *** Process received signal *** +[b05n17:177403] Signal: Segmentation fault (11) +[b05n17:177403] Signal code: (3) +[b05n17:177403] Failing at address: 0x14170000000000a0 +[b11n13:153678] *** Process received signal *** +[b11n13:153678] Signal: Segmentation fault (11) +[b11n13:153678] Signal code: (3) +[b11n13:153678] Failing at address: 0xb170000000000a0 +[b11n13:153678] [ 0] [0x1000000504d8] +[b11n13:153678] [ 1] [0x3fffc12a0cb0] +[b11n13:153678] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n13:153678] [ 3] [b05n18:165696] *** Process received signal *** +[b05n18:165696] Signal: Segmentation fault (11) +[b05n18:165696] Signal code: (3) +[b05n18:165696] Failing at address: 0xb170000000000a0 +[b05n18:165696] [ 0] [0x1000000504d8] +[b05n18:165696] [ 1] [0x3fffef980c60] +[b05n18:165696] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n18:165696] [ 3] [b16n17:119977] *** Process received signal *** +[b16n17:119977] Signal: Segmentation fault (11) +[b16n17:119977] Signal code: (3) +[b16n17:119977] Failing at address: 0x14170000000000a0 +[b16n17:119977] [ 0] [0x1000000504d8] +[b16n17:119977] [ 1] [b23n08:116903] *** Process received signal *** +[b23n08:116903] Signal: Segmentation fault (11) +[b23n08:116903] Signal code: (3) +[b23n08:116903] Failing at address: 0xb170000000000a0 +[b23n08:116903] [ 0] [0x1000000504d8] +[b23n08:116903] [ 1] [0x3fffc686c500] +[b23n08:116903] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n08:116903] [ 3] [b22n04:44755] *** Process received signal *** +[b22n04:44755] Signal: Segmentation fault (11) +[b22n04:44755] Signal code: (3) +[b22n04:44755] Failing at address: 0x14170000000000a0 +[b01n11:03987] *** Process received signal *** +[b01n11:03987] Signal: Segmentation fault (11) +[b01n11:03987] Signal code: (3) +[b01n11:03987] Failing at address: 0x14170000000000a0 +[b01n11:03987] [ 0] [0x1000000504d8] +[b01n11:03987] [ 1] [0x3fffd5e30490] +[b01n11:03987] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n11:03987] [ 3] [b04n16:08386] *** Process received signal *** +[b04n16:08386] Signal: Segmentation fault (11) +[b04n16:08386] Signal code: (3) +[b04n16:08386] Failing at address: 0x14170000000000a0 +[b04n16:08386] [ 0] [0x1000000504d8] +[b04n16:08386] [ 1] [0x3fffd6dafa70] +[b04n16:08386] [ 2] [b22n03:120670] *** Process received signal *** +[b22n03:120670] Signal: Segmentation fault (11) +[b22n03:120670] Signal code: (3) +[b22n03:120670] Failing at address: 0x14170000000000a0 +[b22n03:120670] [ 0] [0x1000000504d8] +[b22n03:120670] [ 1] [a35n16:148189] *** Process received signal *** +[a35n16:148189] Signal: Segmentation fault (11) +[a35n16:148189] Signal code: (3) +[a35n16:148189] Failing at address: 0xb170000000000a0 +[a35n16:148189] [ 0] [0x1000000504d8] +[a35n16:148189] [ 1] [a36n05:139910] *** Process received signal *** +[a28n18:179168] *** Process received signal *** +[a28n18:179168] Signal: Segmentation fault (11) +[a28n18:179168] Signal code: (3) +[a28n18:179168] Failing at address: 0xb170000000000a0 +[a28n18:179168] [ 0] [0x1000000504d8] +[a28n18:179168] [ 1] [0x3ffff09c1f60] +[a28n18:179168] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n18:179168] [ 3] [b14n12:123558] *** Process received signal *** +[b14n12:123558] Signal: Segmentation fault (11) +[b14n12:123558] Signal code: (3) +[b14n12:123558] Failing at address: 0x14170000000000a0 +[b14n12:123558] [ 0] [0x1000000504d8] +[b14n12:123558] [ 1] [0x3fffc8bf1080] +[b14n12:123558] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n12:123558] [ 3] [a29n08:159022] *** Process received signal *** +[a29n08:159022] Signal: Segmentation fault (11) +[a29n08:159022] Signal code: (3) +[a29n08:159022] Failing at address: 0xb170000000000a0 +[a29n08:159022] [ 0] [0x1000000504d8] +[a29n08:159022] [ 1] [0x3ffffd5b8e90] +[a29n08:159022] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n08:159022] [ 3] [b18n13:123055] *** Process received signal *** +[b18n13:123055] Signal: Segmentation fault (11) +[b18n13:123055] Signal code: (3) +[b18n13:123055] Failing at address: 0x14170000000000a0 +[b18n13:123055] [ 0] [0x1000000504d8] +[b18n13:123055] [ 1] [b12n10:140991] *** Process received signal *** +[b12n10:140991] Signal: Segmentation fault (11) +[b12n10:140991] Signal code: (3) +[b12n10:140991] Failing at address: 0x14170000000000a0 +[b12n10:140991] [ 0] [0x1000000504d8] +[b12n10:140991] [ 1] [b14n11:130807] *** Process received signal *** +[b14n11:130807] Signal: Segmentation fault (11) +[b14n11:130807] Signal code: (3) +[b14n11:130807] Failing at address: 0x14170000000000a0 +[b14n11:130807] [ 0] [0x1000000504d8] +[b14n11:130807] [ 1] [b20n18:123413] *** Process received signal *** +[b20n18:123413] Signal: Segmentation fault (11) +[b20n18:123413] Signal code: (3) +[b20n18:123413] Failing at address: 0x14170000000000a0 +[b20n18:123413] [ 0] [0x1000000504d8] +[b20n18:123413] [ 1] [0x3fffee77d090] +[b20n18:123413] [ 2] [a33n06:165303] *** Process received signal *** +[a33n06:165303] Signal: Segmentation fault (11) +[a33n06:165303] Signal code: (3) +[a33n06:165303] Failing at address: 0x5170000000000a0 +[a33n06:165303] [ 0] [0x1000000504d8] +[a33n06:165303] [ 1] [0x3fffc183f250] +[a33n06:165303] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n06:165303] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n06:165303] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n06:165303] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n06:165303] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n06:165303] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n06:165303] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n06:165303] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n06:165303] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n06:165303] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n06:165303] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n06:165303] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n06:165303] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n06:165303] *** End of error message *** +[a28n05:04520] *** Process received signal *** +[a28n05:04520] Signal: Segmentation fault (11) +[a28n05:04520] Signal code: (3) +[a28n05:04520] Failing at address: 0x14170000000000a0 +[a28n05:04520] [ 0] [0x1000000504d8] +[a28n05:04520] [ 1] [b11n16:154184] *** Process received signal *** +[b11n16:154184] Signal: Segmentation fault (11) +[b11n16:154184] Signal code: (3) +[b11n16:154184] Failing at address: 0xb170000000000a0 +[b11n16:154184] [ 0] [0x1000000504d8] +[b11n16:154184] [ 1] [0x3fffd3a190d0] +[b11n16:154184] [ 2] [a34n12:145499] *** Process received signal *** +[a34n12:145499] Signal: Segmentation fault (11) +[a34n12:145499] Signal code: (3) +[a34n12:145499] Failing at address: 0x14170000000000a0 +[a34n12:145499] [ 0] [0x1000000504d8] +[a34n12:145499] [ 1] [0x3fffd9911240] +[a34n12:145499] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n12:145499] [ 3] [b18n15:120132] *** Process received signal *** +[b18n15:120132] Signal: Segmentation fault (11) +[b18n15:120132] Signal code: (3) +[b18n15:120132] Failing at address: 0x14170000000000a0 +[b12n07:140135] *** Process received signal *** +[b12n07:140135] Signal: Segmentation fault (11) +[b12n07:140135] Signal code: (3) +[b12n07:140135] Failing at address: 0x14170000000000a0 +[b12n07:140135] [ 0] [0x1000000504d8] +[b12n07:140135] [ 1] [0x3fffec342860] +[b12n07:140135] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n07:140135] [ 3] [b20n10:131274] *** Process received signal *** +[b20n10:131274] Signal: Segmentation fault (11) +[b20n10:131274] Signal code: (3) +[b20n10:131274] Failing at address: 0xb170000000000a0 +[b20n10:131274] [ 0] [0x1000000504d8] +[b20n10:131274] [ 1] [0x3fffdc80fc60] +[b20n10:131274] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n10:131274] [ 3] [b05n14:177510] *** Process received signal *** +[b05n14:177510] Signal: Segmentation fault (11) +[b05n14:177510] Signal code: (3) +[b05n14:177510] Failing at address: 0xb170000000000a0 +[b05n14:177510] [ 0] [0x1000000504d8] +[b05n14:177510] [ 1] [a34n02:139630] *** Process received signal *** +[a34n02:139630] Signal: Segmentation fault (11) +[a34n02:139630] Signal code: (3) +[a34n02:139630] Failing at address: 0x14170000000000a0 +[a34n02:139630] [ 0] [0x1000000504d8] +[a34n02:139630] [ 1] [b20n11:123521] *** Process received signal *** +[b20n11:123521] Signal: Segmentation fault (11) +[b20n11:123521] Signal code: (3) +[b20n11:123521] Failing at address: 0x14170000000000a0 +[b20n11:123521] [ 0] [0x1000000504d8] +[b20n11:123521] [ 1] [a33n13:168161] *** Process received signal *** +[a33n13:168161] Signal: Segmentation fault (11) +[a33n13:168161] Signal code: (3) +[a33n13:168161] Failing at address: 0x14170000000000a0 +[a33n13:168161] [ 0] [b23n05:120231] *** Process received signal *** +[b23n05:120231] Signal: Segmentation fault (11) +[b23n05:120231] Signal code: (3) +[b23n05:120231] Failing at address: 0x14170000000000a0 +[b23n05:120231] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n10:122761] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n10:122761] [ 6] [b09n09:170658] *** Process received signal *** +[b09n09:170658] Signal: Segmentation fault (11) +[b09n09:170658] Signal code: (3) +[b09n09:170658] Failing at address: 0x14170000000000a0 +[b09n09:170658] [ 0] [0x1000000504d8] +[b09n09:170658] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n14:126120] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n14:126120] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n17:129934] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n17:129934] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n17:129934] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n17:129934] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n12:160187] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n12:160187] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n12:160187] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n12:160187] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n04:06832] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n04:06832] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n04:06832] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n04:06832] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n04:06832] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n04:06832] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n16:124959] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n16:124959] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n09:161233] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n09:161233] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n09:161233] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n09:161233] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n09:161233] *** End of error message *** +[b10n07:116115] *** Process received signal *** +[b10n07:116115] Signal: Segmentation fault (11) +[b10n07:116115] Signal code: (3) +[b10n07:116115] Failing at address: 0x14170000000000a0 +[b10n07:116115] [ 0] [0x1000000504d8] +[b10n07:116115] [ 1] [0x3fffc298adb0] +[b10n07:116115] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n07:116115] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n14:116415] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n14:116415] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n14:116415] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n14:116415] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n02:149229] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n02:149229] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n02:149229] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n02:149229] [10] [0x1000000504d8] +[a27n05:53375] [ 1] [0x3fffcb87ef80] +[a27n05:53375] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n05:53375] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n05:53375] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n05:53375] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n05:53375] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n05:53375] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n05:53375] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n05:53375] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n05:53375] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n08:118552] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n08:118552] *** End of error message *** +[0x1000000504d8] +[b11n07:156144] [ 1] [0x3fffda000510] +[b11n07:156144] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n07:156144] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n07:156144] [ 4] [b18n18:121304] [ 0] [0x1000000504d8] +[b18n18:121304] [ 1] [0x3fffcbd4d350] +[b18n18:121304] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n18:121304] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n18:121304] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n12:134493] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n12:134493] [ 6] [a27n11:23065] *** Process received signal *** +[a27n11:23065] Signal: Segmentation fault (11) +[a27n11:23065] Signal code: (3) +[a27n11:23065] Failing at address: 0x14170000000000a0 +[a27n11:23065] [ 0] [0x1000000504d8] +[a27n11:23065] [ 1] [0x3fffc596b600] +[a27n11:23065] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n11:23065] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n11:23065] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n11:23065] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n11:23065] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n11:23065] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n11:23065] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n11:23065] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n11:23065] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n11:23065] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n11:23065] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n11:23065] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n16:147288] [ 9] [0x3fffeaa940d0] +[a28n12:177495] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n12:177495] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n12:177495] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n12:177495] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n12:177495] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n12:177495] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n12:177495] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n04:172231] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n04:172231] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n04:172231] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n04:172231] [13] [b04n17:08308] *** Process received signal *** +[b04n17:08308] Signal: Segmentation fault (11) +[b04n17:08308] Signal code: (3) +[b04n17:08308] Failing at address: 0x14170000000000a0 +[b04n17:08308] [ 0] [0x1000000504d8] +[b04n17:08308] [ 1] [0x3fffc28bb810] +[b04n17:08308] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n17:08308] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n17:08308] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n04:177596] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n04:177596] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n04:177596] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n04:177596] [12] [0x1000000504d8] +[a27n17:15872] [ 1] [0x3fffffb81420] +[a27n17:15872] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n17:15872] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n17:15872] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n17:15872] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n03:136666] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n03:136666] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n03:136666] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n03:136666] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n03:136666] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n03:136666] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n16:168049] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n16:168049] [ 5] [b02n18:03314] *** Process received signal *** +[b02n18:03314] Signal: Segmentation fault (11) +[b02n18:03314] Signal code: (3) +[b02n18:03314] Failing at address: 0xb170000000000a0 +[b02n18:03314] [ 0] [0x1000000504d8] +[b02n18:03314] [ 1] [0x3ffffe595a60] +[b02n18:03314] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n18:03314] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n18:03314] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n13:131758] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n13:131758] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n13:131758] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n13:131758] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n13:131758] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n01:138008] *** End of error message *** +[b23n04:117355] *** Process received signal *** +[b23n04:117355] Signal: Segmentation fault (11) +[b23n04:117355] Signal code: (3) +[b23n04:117355] Failing at address: 0x14170000000000a0 +[b23n04:117355] [ 0] [0x1000000504d8] +[b23n04:117355] [ 1] [0x3ffff42d38b0] +[b23n04:117355] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n09:143901] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n09:143901] [ 9] [b12n13:130060] *** Process received signal *** +[b12n13:130060] Signal: Segmentation fault (11) +[b12n13:130060] Signal code: (3) +[b12n13:130060] Failing at address: 0x5170000000000a0 +[b12n13:130060] [ 0] [0x1000000504d8] +[b12n13:130060] [ 1] [0x3ffff425b1f0] +[b12n13:130060] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n13:130060] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n13:130060] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n13:130060] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n13:130060] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n09:62480] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n09:62480] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n09:62480] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n09:62480] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n09:62480] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n09:62480] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n09:62480] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n09:62480] [10] [0x3fffeea813a0] +[b12n14:139846] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n14:139846] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n14:139846] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n14:139846] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n13:123518] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n13:123518] *** End of error message *** +[a33n07:149478] *** Process received signal *** +[a33n07:149478] Signal: Segmentation fault (11) +[a33n07:149478] Signal code: (3) +[a33n07:149478] Failing at address: 0x14170000000000a0 +[a33n07:149478] [ 0] [0x1000000504d8] +[a33n07:149478] [ 1] [0x3ffff3dda5d0] +[a33n07:149478] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n07:149478] [ 3] [b09n10:169684] *** Process received signal *** +[b09n10:169684] Signal: Segmentation fault (11) +[b09n10:169684] Signal code: (3) +[b09n10:169684] Failing at address: 0xb170000000000a0 +[b09n10:169684] [ 0] [0x1000000504d8] +[b09n10:169684] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n17:168164] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n17:168164] *** End of error message *** +[b20n14:120639] *** Process received signal *** +[b20n14:120639] Signal: Segmentation fault (11) +[b20n14:120639] Signal code: (3) +[b20n14:120639] Failing at address: 0xb170000000000a0 +[b20n14:120639] [ 0] [0x1000000504d8] +[b20n14:120639] [ 1] [0x3fffffb60b60] +[b20n14:120639] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n13:122187] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n13:122187] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n13:122187] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n13:122187] [ 8] [a28n15:07536] *** Process received signal *** +[a28n15:07536] Signal: Segmentation fault (11) +[a28n15:07536] Signal code: (3) +[a28n15:07536] Failing at address: 0x14170000000000a0 +[a28n15:07536] [ 0] [0x1000000504d8] +[a28n15:07536] [ 1] [0x3ffff1f09950] +[a28n15:07536] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n15:07536] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n15:07536] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n15:07536] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n15:07536] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n15:07536] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n15:07536] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n15:07536] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n15:07536] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n10:126192] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n10:126192] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n10:126192] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n10:126192] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n10:126192] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n10:126192] [14] [0x1000000504d8] +[b13n08:144853] [ 1] [0x3ffffda00960] +[b13n08:144853] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n08:144853] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n08:144853] [ 4] [ 0] [0x1000000504d8] +[b06n12:177118] [ 1] [0x3fffceda2220] +[b06n12:177118] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n12:177118] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n12:177118] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n12:177118] [ 5] [b16n09:124192] *** Process received signal *** +[b16n09:124192] Signal: Segmentation fault (11) +[b16n09:124192] Signal code: (3) +[b16n09:124192] Failing at address: 0x14170000000000a0 +[b16n09:124192] [ 0] [0x1000000504d8] +[b16n09:124192] [ 1] [0x3fffdc6738c0] +[b16n09:124192] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n09:124192] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n13:177640] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n13:177640] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n13:177640] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n13:177640] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n13:177640] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n13:177640] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n13:177640] [10] [b17n17:125808] *** Process received signal *** +[b17n17:125808] Signal: Segmentation fault (11) +[b17n17:125808] Signal code: (3) +[b17n17:125808] Failing at address: 0x14170000000000a0 +[b17n17:125808] [ 0] [0x1000000504d8] +[b17n17:125808] [ 1] [0x3fffc8a2a040] +[b17n17:125808] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n17:125808] [ 3] [b23n12:115636] *** Process received signal *** +[b23n12:115636] Signal: Segmentation fault (11) +[b23n12:115636] Signal code: (3) +[b23n12:115636] Failing at address: 0xb170000000000a0 +[b23n12:115636] [ 0] [a36n11:152843] *** Process received signal *** +[a36n11:152843] Signal: Segmentation fault (11) +[a36n11:152843] Signal code: (3) +[a36n11:152843] Failing at address: 0xb170000000000a0 +[a36n11:152843] [ 0] [0x1000000504d8] +[a36n11:152843] [ 1] [0x3fffc27c01b0] +[a36n11:152843] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n11:152843] [ 3] [b06n07:174561] *** Process received signal *** +[b06n07:174561] Signal: Segmentation fault (11) +[b06n07:174561] Signal code: (3) +[b06n07:174561] Failing at address: 0x14170000000000a0 +[b06n07:174561] [ 0] [0x1000000504d8] +[b06n07:174561] [ 1] [0x3fffc913a190] +[b06n07:174561] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n07:174561] [ 3] [b02n12:180117] *** Process received signal *** +[b02n12:180117] Signal: Segmentation fault (11) +[b02n12:180117] Signal code: (3) +[b02n12:180117] Failing at address: 0x14170000000000a0 +[b02n12:180117] [ 0] [0x1000000504d8] +[b02n12:180117] [ 1] [0x3fffc00ae230] +[b02n12:180117] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n12:180117] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n07:173042] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n07:173042] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n07:173042] [ 6] [a31n13:155852] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n13:133628] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n13:133628] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n13:138089] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n13:138089] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n13:138089] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n13:138089] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n15:32401] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n08:131378] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n08:141875] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n08:141875] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n12:164384] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n12:164384] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n12:164384] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n12:164384] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n12:164384] [14] [b09n01:163024] *** Process received signal *** +[b09n01:163024] Signal: Segmentation fault (11) +[b09n01:163024] Signal code: (3) +[b09n01:163024] Failing at address: 0xb170000000000a0 +[b09n01:163024] [ 0] [0x1000000504d8] +[b09n01:163024] [ 1] [0x3fffcafafb10] +[b09n01:163024] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n01:163024] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n01:163024] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n01:163024] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n13:163196] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n18:176810] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n06:164017] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n06:164017] *** End of error message *** +[11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n12:120402] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n12:120402] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n12:120402] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n12:120402] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n18:164801] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n02:167597] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n14:08400] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n14:08400] *** End of error message *** +[b06n18:173979] *** Process received signal *** +[b06n18:173979] Signal: Segmentation fault (11) +[b06n18:173979] Signal code: (3) +[b06n18:173979] Failing at address: 0x14170000000000a0 +[b06n18:173979] [ 0] [0x1000000504d8] +[b06n18:173979] [ 1] [0x3ffff4c701c0] +[b06n18:173979] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n18:173979] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n14:119484] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n14:119484] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n14:119484] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n14:119484] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n14:119484] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n11:162471] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n11:162471] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n11:162471] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n11:162471] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n15:119252] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n15:119252] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n15:119252] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n14:150132] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n14:150132] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n14:150132] [14] [b11n04:166206] *** Process received signal *** +[b11n04:166206] Signal: Segmentation fault (11) +[b11n04:166206] Signal code: (3) +[b11n04:166206] Failing at address: 0xb170000000000a0 +[b11n04:166206] [ 0] [b05n07:08397] *** Process received signal *** +[b05n07:08397] Signal: Segmentation fault (11) +[b05n07:08397] Signal code: (3) +[b05n07:08397] Failing at address: 0x14170000000000a0 +[b05n07:08397] [ 0] [0x1000000504d8] +[b05n07:08397] [ 1] [0x3fffc2a5d6f0] +[b05n07:08397] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n07:08397] [ 3] [b23n10:126793] *** Process received signal *** +[b23n10:126793] Signal: Segmentation fault (11) +[b23n10:126793] Signal code: (3) +[b23n10:126793] Failing at address: 0x14170000000000a0 +[b23n10:126793] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n10:118388] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n10:118388] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n10:118388] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n10:118388] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n10:118388] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n14:173894] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n14:173894] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n14:173894] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n14:173894] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n14:173894] [13] [0x1000000504d8] +[b21n13:117510] [ 1] [0x3ffff499b890] +[b21n13:117510] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n13:117510] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n13:117510] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n14:136660] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n14:136660] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n14:136660] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n15:154384] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n15:154384] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n15:154384] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n15:154384] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n10:134735] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n10:134735] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n10:134735] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n10:134735] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n09:123815] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n09:123815] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n09:123815] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n09:123815] [14] [b02n17:03846] *** Process received signal *** +[b02n17:03846] Signal: Segmentation fault (11) +[b02n17:03846] Signal code: (3) +[b02n17:03846] Failing at address: 0xb170000000000a0 +[b02n17:03846] [ 0] [0x1000000504d8] +[b02n17:03846] [ 1] [0x3fffe8898030] +[b02n17:03846] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n17:03846] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n17:03846] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n18:08432] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n18:08432] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n18:08432] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n18:08432] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n18:08432] *** End of error message *** +[0x3fffd1528850] +[b09n16:173548] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n16:173548] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n16:173548] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n16:173548] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n11:163262] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n11:163262] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n11:163262] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n11:163262] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n11:160797] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n11:160797] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n11:160797] [ 8] [a32n04:153966] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n05:154190] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n05:154190] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n05:154190] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n05:154190] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n05:154190] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n05:154190] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n05:154190] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n04:133905] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n04:133905] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n04:133905] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n04:133905] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n04:133905] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n04:133905] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n03:158660] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n03:158660] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n03:158660] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n14:21318] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n14:21318] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n14:21318] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n14:21318] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n06:135851] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n06:135851] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n06:135851] [ 6] [0x3ffff87cf6a0] +[b21n12:120130] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n12:120130] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n12:120130] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n18:140388] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n18:140388] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n18:177223] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n18:177223] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n18:177223] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n16:149064] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n16:149064] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n16:149064] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n16:149064] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n10:173683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n10:173683] [ 5] [ 0] [0x1000000504d8] +[a31n11:153420] [ 1] [0x3fffe40c6520] +[a31n11:153420] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n11:153420] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n11:153420] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n07:125449] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n07:125449] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n07:125449] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n05:138197] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n05:138197] [ 5] [b12n09:128000] *** Process received signal *** +[b12n09:128000] Signal: Segmentation fault (11) +[b12n09:128000] Signal code: (3) +[b12n09:128000] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n18:138475] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n18:138475] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n07:133377] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n07:133377] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n07:133377] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n03:119832] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n17:171200] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n17:171200] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n17:171200] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n17:171200] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n17:171200] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n05:54536] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n05:54536] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n05:54536] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n05:54536] [ 8] [a29n18:07423] *** Process received signal *** +[a29n18:07423] Signal: Segmentation fault (11) +[a29n18:07423] Signal code: (3) +[a29n18:07423] Failing at address: 0x14170000000000a0 +[a29n18:07423] [ 0] [0x1000000504d8] +[a29n18:07423] [ 1] [0x3ffffb2e2f90] +[a29n18:07423] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n18:07423] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n18:07423] [ 4] [b07n16:166498] *** Process received signal *** +[b07n16:166498] Signal: Segmentation fault (11) +[b07n16:166498] Signal code: (3) +[b07n16:166498] Failing at address: 0x14170000000000a0 +[b07n16:166498] [ 0] [0x1000000504d8] +[b07n16:166498] [ 1] [0x3ffff5e2ed90] +[b07n16:166498] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n16:166498] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n01:121744] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n01:121744] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n01:121744] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n01:121744] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n01:121744] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n01:121744] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n01:121744] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n01:121744] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n01:121744] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n01:121744] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n01:121744] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n01:121744] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n09:168246] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n09:168246] *** End of error message *** +[b21n08:116172] *** Process received signal *** +[b21n08:116172] Signal: Segmentation fault (11) +[b21n08:116172] Signal code: (3) +[b21n08:116172] Failing at address: 0xb170000000000a0 +[b21n08:116172] [ 0] [a27n04:52705] *** Process received signal *** +[a27n04:52705] Signal: Segmentation fault (11) +[a27n04:52705] Signal code: (3) +[a27n04:52705] Failing at address: 0x14170000000000a0 +[a27n04:52705] [ 0] [0x1000000504d8] +[a27n04:52705] [ 1] [0x3fffd05fb850] +[a27n04:52705] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n04:52705] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n04:52705] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n04:52705] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n04:52705] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n04:52705] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n04:52705] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n04:52705] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n04:52705] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n04:52705] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n04:52705] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n04:52705] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n16:165922] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n16:165922] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n16:127354] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n16:127354] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n16:127354] [10] [ 1] [0x3fffdbb1a890] +[b21n07:117359] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n07:117359] [b18n17:124167] *** Process received signal *** +[b18n17:124167] Signal: Segmentation fault (11) +[b18n17:124167] Signal code: (3) +[b18n17:124167] Failing at address: 0x14170000000000a0 +[b18n17:124167] [ 0] [0x1000000504d8] +[b18n17:124167] [ 1] [0x3fffead6d9a0] +[b18n17:124167] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n17:124167] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n07:124350] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n13:122031] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n13:122031] *** End of error message *** +[b16n11:124435] *** Process received signal *** +[b16n11:124435] Signal: Segmentation fault (11) +[b16n11:124435] Signal code: (3) +[b16n11:124435] Failing at address: 0xb170000000000a0 +[b16n11:124435] [ 0] [0x1000000504d8] +[b16n11:124435] [ 1] [0x3ffffa793960] +[b16n11:124435] [ 2] [0x3fffe1cd32b0] +[b13n16:128443] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n16:128443] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n16:128443] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n16:128443] [ 5] [a26n18:69155] *** Process received signal *** +[a26n18:69155] Signal: Segmentation fault (11) +[a26n18:69155] Signal code: (3) +[a26n18:69155] Failing at address: 0x14170000000000a0 +[a26n18:69155] [ 0] [0x1000000504d8] +[a26n18:69155] [ 1] [b03n14:177225] *** Process received signal *** +[b03n14:177225] Signal: Segmentation fault (11) +[b03n14:177225] Signal code: (3) +[b03n14:177225] Failing at address: 0x14170000000000a0 +[b03n14:177225] [ 0] [0x1000000504d8] +[b03n14:177225] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n14:08951] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n14:08951] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n10:124502] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n10:124502] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n10:124502] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n10:124502] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n10:124502] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n10:124502] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n03:38501] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n03:38501] [ 5] [a27n01:48805] *** Process received signal *** +[a27n01:48805] Signal: Segmentation fault (11) +[a27n01:48805] Signal code: (3) +[a27n01:48805] Failing at address: 0x14170000000000a0 +[a27n01:48805] [ 0] [b08n12:12761] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n12:167678] [ 4] [0x3fffffc5e440] +[a29n15:177059] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n15:177059] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n15:177059] [ 4] [0x3fffffc91700] +[b23n13:114397] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n13:114397] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n13:114397] [ 4] [ 1] [0x3fffec7b6ca0] +[a36n10:142929] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n10:142929] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n10:142929] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n10:142929] [ 5] [0x1000000504d8] +[b11n14:161616] [ 1] [0x3ffff2f8a9e0] +[b11n14:161616] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n14:161616] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n14:161616] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n11:160346] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n11:160346] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n17:149904] [ 5] [b21n16:116526] *** Process received signal *** +[b21n16:116526] Signal: Segmentation fault (11) +[b21n16:116526] Signal code: (3) +[b21n16:116526] Failing at address: 0x14170000000000a0 +[b21n16:116526] [ 0] [0x1000000504d8] +[b21n16:116526] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n05:135164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n05:135164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n05:135164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n05:135164] [14] [a29n16:157889] *** Process received signal *** +[a29n16:157889] Signal: Segmentation fault (11) +[a29n16:157889] Signal code: (3) +[a29n16:157889] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n03:177924] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n03:177924] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n03:177924] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n03:177924] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n03:177924] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n13:165390] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n13:165390] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n13:165390] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n13:165390] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n13:165390] *** End of error message *** +[b23n07:118167] *** Process received signal *** +[b23n07:118167] Signal: Segmentation fault (11) +[b23n07:118167] Signal code: (3) +[b23n07:118167] Failing at address: 0x14170000000000a0 +[b23n07:118167] [ 0] [0x1000000504d8] +[b23n07:118167] [ 1] [0x3fffdcad0d70] +[b23n07:118167] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n07:118167] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n09:03928] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n09:03928] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n09:03928] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n14:128872] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n11:62421] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n11:62421] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n11:62421] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n11:62421] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n11:62421] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n11:62421] [ 8] [0x1000000504d8] +[a28n10:176311] [ 1] [0x3fffdeb93b10] +[a28n10:176311] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n10:176311] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n10:176311] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n14:169524] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n14:169524] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n14:169524] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n14:169524] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n14:169524] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n14:169524] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n14:169524] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n10:10924] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n10:10924] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n14:133633] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n06:127277] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n06:127277] [ 6] [b01n18:179699] *** Process received signal *** +[b01n18:179699] Signal: Segmentation fault (11) +[b01n18:179699] Signal code: (3) +[b01n18:179699] Failing at address: 0x5170000000000a0 +[b01n18:179699] [ 0] [0x1000000504d8] +[b09n11:171963] [ 1] [0x3fffd2dd4140] +[b09n11:171963] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n11:171963] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n11:171963] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n11:171963] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n02:119535] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n02:119535] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n01:141816] [ 5] [0x3fffce4bcc30] +[b20n16:124665] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n16:124665] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n16:124665] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n06:143011] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n06:143011] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n06:143011] [ 6] [a36n07:153177] *** Process received signal *** +[a36n07:153177] Signal: Segmentation fault (11) +[a36n07:153177] Signal code: (3) +[a36n07:153177] Failing at address: 0x5170000000000a0 +[a36n07:153177] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n13:156920] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n13:156920] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n08:147821] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n08:147821] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n04:169911] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n04:169911] [ 6] [a27n18:28221] *** Process received signal *** +[a27n18:28221] Signal: Segmentation fault (11) +[a27n18:28221] Signal code: (3) +[a27n18:28221] Failing at address: 0xb170000000000a0 +[a27n18:28221] [ 0] [0x1000000504d8] +[a27n18:28221] [ 1] [0x3fffca57d670] +[a27n18:28221] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n18:28221] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n18:28221] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n08:08957] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n02:164493] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n02:164493] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n02:164493] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n02:164493] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n13:176368] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n13:176368] *** End of error message *** +[b11n11:152988] *** Process received signal *** +[b11n11:152988] Signal: Segmentation fault (11) +[b11n11:152988] Signal code: (3) +[b11n11:152988] Failing at address: 0xb170000000000a0 +[b11n11:152988] [ 0] [0x1000000504d8] +[b11n11:152988] [ 1] [0x3fffe884d980] +[b11n11:152988] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n11:152988] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n13:123707] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n13:123707] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n13:123707] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n18:126908] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n18:126908] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n18:126908] [ 9] [b20n07:132090] *** Process received signal *** +[b20n07:132090] Signal: Segmentation fault (11) +[b20n07:132090] Signal code: (3) +[b20n07:132090] Failing at address: 0x14170000000000a0 +[b20n07:132090] [ 0] [0x1000000504d8] +[b20n07:132090] [ 1] [0x3fffcfb982d0] +[b20n07:132090] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n07:132090] [ 3] [a26n10:67858] *** Process received signal *** +[a26n10:67858] Signal: Segmentation fault (11) +[a26n10:67858] Signal code: (3) +[a26n10:67858] Failing at address: 0x14170000000000a0 +[a26n10:67858] [ 0] [0x1000000504d8] +[a26n10:67858] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n04:126859] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n04:126859] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n04:126859] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n04:126859] [13] [b03n15:85939] *** Process received signal *** +[b03n15:85939] Signal: Segmentation fault (11) +[b03n15:85939] Signal code: (3) +[b03n15:85939] Failing at address: 0xb170000000000a0 +[b03n15:85939] [ 0] [0x1000000504d8] +[b03n15:85939] [ 1] [0x3ffffcf245f0] +[b03n15:85939] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n15:85939] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n09:120695] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n09:120695] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n09:120695] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n09:120695] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n09:120695] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n09:120695] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n09:120695] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n09:120695] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n09:120695] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n09:120695] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n09:120695] *** End of error message *** +[b02n16:01571] *** Process received signal *** +[b02n16:01571] Signal: Segmentation fault (11) +[b02n16:01571] Signal code: (3) +[b02n16:01571] Failing at address: 0xb170000000000a0 +[b02n16:01571] [ 0] [0x1000000504d8] +[b02n16:01571] [ 1] [0x3fffe5a6b870] +[b02n16:01571] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n16:01571] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n09:173750] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n09:173750] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n09:173750] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n10:130582] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n10:130582] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n10:130582] [ 6] [b15n09:121754] *** Process received signal *** +[b15n09:121754] Signal: Segmentation fault (11) +[b15n09:121754] Signal code: (3) +[b15n09:121754] Failing at address: 0x14170000000000a0 +[a28n03:10291] *** Process received signal *** +[a28n03:10291] Signal: Segmentation fault (11) +[a28n03:10291] Signal code: (3) +[a28n03:10291] Failing at address: 0x14170000000000a0 +[a28n03:10291] [ 0] [0x1000000504d8] +[a28n03:10291] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n14:168376] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n14:168376] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n14:168376] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n14:168376] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n14:168376] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n14:168376] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n14:168376] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n14:168376] [13] [0x1000000504d8] +[a28n11:01170] [ 1] [0x3fffc1d90bc0] +[a28n11:01170] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n11:01170] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n11:01170] [ 4] [b20n08:124726] *** Process received signal *** +[b20n08:124726] Signal: Segmentation fault (11) +[b20n08:124726] Signal code: (3) +[b20n08:124726] Failing at address: 0x14170000000000a0 +[b20n08:124726] [ 0] [0x1000000504d8] +[b20n08:124726] [ 1] [0x3fffc9e7f340] +[b20n08:124726] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n08:124726] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n09:119547] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n13:01302] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n13:01302] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n13:01302] *** End of error message *** +[a30n16:153293] *** Process received signal *** +[a30n16:153293] Signal: Segmentation fault (11) +[a30n16:153293] Signal code: (3) +[a30n16:153293] Failing at address: 0x14170000000000a0 +[a30n16:153293] [ 0] [0x1000000504d8] +[a30n16:153293] [ 1] [0x3fffda2a9600] +[a30n16:153293] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n16:171919] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n15:177862] [10] [b12n11:137873] *** Process received signal *** +[b12n11:137873] Signal: Segmentation fault (11) +[b12n11:137873] Signal code: (3) +[b12n11:137873] Failing at address: 0x5170000000000a0 +[b12n11:137873] [ 0] [0x3fffc434fcc0] +[a26n15:55850] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n15:55850] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n15:55850] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n15:55850] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n15:55850] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n15:55850] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n15:55850] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n15:55850] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n15:55850] [10] [b06n08:174411] *** Process received signal *** +[b06n08:174411] Signal: Segmentation fault (11) +[b06n08:174411] Signal code: (3) +[b06n08:174411] Failing at address: 0x14170000000000a0 +[b06n08:174411] [ 0] [0x1000000504d8] +[b06n08:174411] [ 1] [0x3fffff4e47d0] +[b06n08:174411] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n08:174411] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n17:150932] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n17:150932] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n17:150932] [13] [b19n10:120976] *** Process received signal *** +[b19n10:120976] Signal: Segmentation fault (11) +[b19n10:120976] Signal code: (3) +[b19n10:120976] Failing at address: 0xb170000000000a0 +[b19n10:120976] [ 0] [a27n02:49107] *** Process received signal *** +[a27n02:49107] Signal: Segmentation fault (11) +[a27n02:49107] Signal code: (3) +[a27n02:49107] Failing at address: 0x14170000000000a0 +[a27n02:49107] [ 0] [0x1000000504d8] +[a27n02:49107] [ 1] [0x3fffee92b660] +[a27n02:49107] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n02:49107] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n11:02686] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n11:02686] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n11:02686] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n11:02686] [14] [b15n17:126526] *** Process received signal *** +[b15n17:126526] Signal: Segmentation fault (11) +[b15n17:126526] Signal code: (3) +[b15n17:126526] Failing at address: 0xb170000000000a0 +[b15n17:126526] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n14:163551] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n14:163551] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n14:163551] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n14:163551] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n14:163551] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n15:161758] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n15:161758] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n15:161758] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n15:161758] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n15:161758] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n08:127444] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n08:127444] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n04:13082] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n13:06928] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n13:06928] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n13:06928] [ 9] [b06n17:163817] *** Process received signal *** +[b06n17:163817] Signal: Segmentation fault (11) +[b06n17:163817] Signal code: (3) +[b06n17:163817] Failing at address: 0x14170000000000a0 +[b06n17:163817] [ 0] [0x1000000504d8] +[b06n17:163817] [ 1] [0x3fffcac396a0] +[b06n17:163817] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n09:147792] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n09:147792] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n09:147792] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n09:147792] [13] [b22n16:124798] *** Process received signal *** +[b22n16:124798] Signal: Segmentation fault (11) +[b22n16:124798] Signal code: (3) +[b22n16:124798] Failing at address: 0x14170000000000a0 +[b22n16:124798] [ 0] [0x1000000504d8] +[b22n16:124798] [ 1] [0x3fffd0216f40] +[b22n16:124798] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n16:124798] [ 3] [0x1000000504d8] +[a30n01:163389] [ 1] [0x3fffde9bbeb0] +[a30n01:163389] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n01:163389] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n01:163389] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n03:125239] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n03:125239] *** End of error message *** +[a32n06:154622] *** Process received signal *** +[a32n06:154622] Signal: Segmentation fault (11) +[a32n06:154622] Signal code: (3) +[a32n06:154622] Failing at address: 0x5170000000000a0 +[a32n06:154622] [ 0] [b11n05:162565] *** Process received signal *** +[b11n05:162565] Signal: Segmentation fault (11) +[b11n05:162565] Signal code: (3) +[b11n05:162565] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n11:148673] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n11:148673] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n01:164962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n01:164962] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n01:164962] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n01:164962] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n06:130544] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n06:130544] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n06:130544] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n06:130544] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n08:172428] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n08:172428] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n18:10725] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n18:10725] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n02:122557] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n02:122557] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n02:122557] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n02:122557] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n02:122557] [11] [b08n11:164996] *** Process received signal *** +[b08n11:164996] Signal: Segmentation fault (11) +[b08n11:164996] Signal code: (3) +[b08n11:164996] Failing at address: 0x14170000000000a0 +[b08n11:164996] [ 0] [0x1000000504d8] +[b08n11:164996] [ 1] [0x3fffc28675b0] +[b08n11:164996] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n11:164996] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n03:151884] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n03:151884] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n03:151884] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n03:151884] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n03:151884] [13] [a28n08:175808] *** Process received signal *** +[a28n08:175808] Signal: Segmentation fault (11) +[a28n08:175808] Signal code: (3) +[a28n08:175808] Failing at address: 0x5170000000000a0 +[a28n08:175808] [ 0] [0x1000000504d8] +[a28n08:175808] [ 1] [0x3ffff0575d90] +[a28n08:175808] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n08:175808] [ 3] [b11n10:161719] *** Process received signal *** +[b11n10:161719] Signal: Segmentation fault (11) +[b11n10:161719] Signal code: (3) +[b11n10:161719] Failing at address: 0xb170000000000a0 +[b11n10:161719] [ 0] [0x1000000504d8] +[b11n10:161719] [ 1] [0x3fffec70d7b0] +[b11n10:161719] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n10:161719] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n10:161719] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n15:155431] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n15:155431] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n06:162918] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n06:162918] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n06:162918] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n06:162918] [10] [a29n10:164940] *** Process received signal *** +[a29n10:164940] Signal: Segmentation fault (11) +[a29n10:164940] Signal code: (3) +[a29n10:164940] Failing at address: 0x14170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n05:173848] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n05:173848] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n05:173848] [14] [b05n09:106789] *** Process received signal *** +[b05n09:106789] Signal: Segmentation fault (11) +[b05n09:106789] Signal code: (3) +[b05n09:106789] Failing at address: 0x14170000000000a0 +[b05n09:106789] [ 0] [0x1000000504d8] +[b05n09:106789] [ 1] [0x3fffe52d6c50] +[b05n09:106789] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n09:106789] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n06:133957] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n06:133957] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n06:133957] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n01:171787] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n01:171787] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n01:171787] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n01:171787] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n01:171787] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n05:120331] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n05:120331] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n05:120331] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n05:120331] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n05:120331] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n08:129890] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n08:129890] *** End of error message *** +[b10n12:165844] *** Process received signal *** +[b10n12:165844] Signal: Segmentation fault (11) +[b10n12:165844] Signal code: (3) +[b10n12:165844] Failing at address: 0x14170000000000a0 +[b10n12:165844] [ 0] [0x1000000504d8] +[b10n12:165844] [ 1] [0x3fffe6b40b70] +[b10n12:165844] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n12:165844] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n12:165844] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n12:165844] [ 5] [b08n08:164751] *** Process received signal *** +[b08n08:164751] Signal: Segmentation fault (11) +[b08n08:164751] Signal code: (3) +[b08n08:164751] Failing at address: 0x14170000000000a0 +[b08n08:164751] [ 0] [b06n06:163677] *** Process received signal *** +[b06n06:163677] Signal: Segmentation fault (11) +[b06n06:163677] Signal code: (3) +[b06n06:163677] Failing at address: 0x14170000000000a0 +[b06n06:163677] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n14:126466] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n14:126466] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n14:126466] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n14:126466] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n12:156916] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n15:177822] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n15:177822] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n15:177822] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n15:177822] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n15:177822] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n15:177822] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n16:164551] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n16:164551] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n05:173082] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n05:173082] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n05:173082] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n02:158418] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n02:158418] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n02:158418] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n02:158418] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n02:158418] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n02:158418] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n02:158418] [13] [b15n01:123574] *** Process received signal *** +[b15n01:123574] Signal: Segmentation fault (11) +[b15n01:123574] Signal code: (3) +[b15n01:123574] Failing at address: 0x14170000000000a0 +[b15n01:123574] [ 0] [0x1000000504d8] +[b15n01:123574] [ 1] [0x3fffc12ebf70] +[b15n01:123574] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n03:175065] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n03:175065] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n03:175065] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n03:175065] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n03:175065] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n03:175065] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n03:175065] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n03:175065] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n03:175065] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n03:175065] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n03:175065] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n07:60173] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n07:60173] [ 5] [b08n13:158865] *** Process received signal *** +[b08n13:158865] Signal: Segmentation fault (11) +[b08n13:158865] Signal code: (3) +[b08n13:158865] Failing at address: 0x14170000000000a0 +[b08n13:158865] [ 0] [0x1000000504d8] +[b08n13:158865] [ 1] [0x3fffe332b520] +[b08n13:158865] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n13:158865] [ 3] [b12n12:138738] *** Process received signal *** +[b12n12:138738] Signal: Segmentation fault (11) +[b12n12:138738] Signal code: (3) +[b12n12:138738] Failing at address: 0x5170000000000a0 +[b12n12:138738] [ 0] [0x1000000504d8] +[b12n12:138738] [ 1] [0x3ffff252a120] +[b12n12:138738] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n12:138738] [ 3] [a32n09:155756] *** Process received signal *** +[a32n09:155756] Signal: Segmentation fault (11) +[a32n09:155756] Signal code: (3) +[a32n09:155756] Failing at address: 0xb170000000000a0 +[a32n09:155756] [ 0] [0x1000000504d8] +[a32n09:155756] [ 1] [0x3ffff843f7b0] +[a32n09:155756] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n09:155756] [ 3] [b19n17:123238] *** Process received signal *** +[b19n17:123238] Signal: Segmentation fault (11) +[b19n17:123238] Signal code: (3) +[b19n17:123238] Failing at address: 0x14170000000000a0 +[b19n17:123238] [ 0] [0x1000000504d8] +[b19n17:123238] [ 1] [0x3fffdb03d780] +[b19n17:123238] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n17:123238] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n17:123238] [ 4] [a29n17:169738] *** Process received signal *** +[a29n17:169738] Signal: Segmentation fault (11) +[a29n17:169738] Signal code: (3) +[a29n17:169738] Failing at address: 0x14170000000000a0 +[a29n17:169738] [ 0] [0x1000000504d8] +[a29n17:169738] [ 1] [0x3fffca0bb560] +[a29n17:169738] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n17:168156] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n17:168156] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n17:168156] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n17:168156] [ 8] [b23n06:117526] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n07:152085] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n07:152085] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n07:152085] [ 6] [b10n01:165124] *** Process received signal *** +[b10n01:165124] Signal: Segmentation fault (11) +[b10n01:165124] Signal code: (3) +[b10n01:165124] Failing at address: 0xb170000000000a0 +[b10n01:165124] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n06:138767] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n06:138767] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n06:138767] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n06:138767] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n06:138767] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n06:138767] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n06:138767] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n06:161725] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n06:161725] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n06:161725] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n06:161725] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n06:161725] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n06:161725] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n06:161725] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n06:161725] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n06:161725] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n06:161725] *** End of error message *** +[b11n03:154296] *** Process received signal *** +[b11n03:154296] Signal: Segmentation fault (11) +[b11n03:154296] Signal code: (3) +[b11n03:154296] Failing at address: 0x14170000000000a0 +[b11n03:154296] [ 0] [0x1000000504d8] +[b11n03:154296] [ 1] [0x3fffc2ed8500] +[b11n03:154296] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n03:154296] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n03:154296] [ 4] [a36n12:141260] *** Process received signal *** +[a36n12:141260] Signal: Segmentation fault (11) +[a36n12:141260] Signal code: (3) +[a36n12:141260] Failing at address: 0x14170000000000a0 +[a36n12:141260] [ 0] [0x1000000504d8] +[a36n12:141260] [ 1] [0x3ffffde7c6a0] +[a36n12:141260] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n12:141260] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n12:178596] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n12:178596] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n14:178324] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n14:178324] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n07:156491] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n07:156491] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n07:156491] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n07:156491] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n07:156491] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n18:118682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n18:118682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n18:118682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n18:118682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n18:118682] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n18:118682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n18:118682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n18:118682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n18:118682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n18:118682] [14] [b23n11:117476] *** Process received signal *** +[b23n11:117476] Signal: Segmentation fault (11) +[b23n11:117476] Signal code: (3) +[b23n11:117476] Failing at address: 0x14170000000000a0 +[b23n11:117476] [ 0] [0x1000000504d8] +[b23n11:117476] [ 1] [0x3fffc1ecd2d0] +[b23n11:117476] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n04:158633] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n04:158633] [10] [b08n07:168964] *** Process received signal *** +[b08n07:168964] Signal: Segmentation fault (11) +[b08n07:168964] Signal code: (3) +[b08n07:168964] Failing at address: 0x14170000000000a0 +[b08n07:168964] [ 0] [0x1000000504d8] +[b08n07:168964] [ 1] [0x3fffe7f22bf0] +[b08n07:168964] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n07:168964] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n07:168964] [ 4] [a29n06:157790] *** Process received signal *** +[a29n06:157790] Signal: Segmentation fault (11) +[a29n06:157790] Signal code: (3) +[a29n06:157790] Failing at address: 0x14170000000000a0 +[a29n06:157790] [ 0] [0x1000000504d8] +[a29n06:157790] [ 1] [0x3ffff67cb140] +[a29n06:157790] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n06:157790] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n06:157790] [ 4] [0x3ffffc1b83f0] +[a35n04:145750] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n04:145750] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n04:145750] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n04:145750] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n17:176827] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n17:176827] [ 4] [0x1000000504d8] +[b21n11:120580] [ 1] [0x3ffffb3856d0] +[b21n11:120580] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n11:120580] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n11:120580] [ 4] [b23n09:119635] *** Process received signal *** +[b23n09:119635] Signal: Segmentation fault (11) +[b23n09:119635] Signal code: (3) +[b23n09:119635] Failing at address: 0x14170000000000a0 +[b23n09:119635] [ 0] [0x1000000504d8] +[b23n09:119635] [ 1] [b02n13:171110] *** Process received signal *** +[b02n13:171110] Signal: Segmentation fault (11) +[b02n13:171110] Signal code: (3) +[b02n13:171110] Failing at address: 0x14170000000000a0 +[b02n13:171110] [ 0] [0x1000000504d8] +[b02n13:171110] [ 1] [b08n10:11488] *** Process received signal *** +[b08n10:11488] Signal: Segmentation fault (11) +[b08n10:11488] Signal code: (3) +[b08n10:11488] Failing at address: 0x14170000000000a0 +[b08n10:11488] [ 0] [0x1000000504d8] +[b08n10:11488] [ 1] [0x3fffd7fba310] +[b08n10:11488] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n10:11488] [ 3] [a29n07:160803] *** Process received signal *** +[a29n07:160803] Signal: Segmentation fault (11) +[a29n07:160803] Signal code: (3) +[a29n07:160803] Failing at address: 0x14170000000000a0 +[a29n07:160803] [ 0] [0x1000000504d8] +[a29n07:160803] [ 1] [0x3fffc127edb0] +[a29n07:160803] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n07:160803] [ 3] [a27n03:46687] *** Process received signal *** +[a27n03:46687] Signal: Segmentation fault (11) +[a27n03:46687] Signal code: (3) +[a27n03:46687] Failing at address: 0xb170000000000a0 +[a27n03:46687] [ 0] [0x1000000504d8] +[a27n03:46687] [ 1] [0x3fffffe4d2b0] +[a27n03:46687] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n03:46687] [ 3] [b05n17:177404] *** Process received signal *** +[b05n17:177404] Signal: Segmentation fault (11) +[b05n17:177404] Signal code: (3) +[b05n17:177404] Failing at address: 0x14170000000000a0 +[b05n17:177404] [ 0] [0x1000000504d8] +[b05n17:177404] [ 1] [0x3fffd6567fc0] +[b05n17:177404] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n17:177404] [ 3] [b11n13:153679] *** Process received signal *** +[b11n13:153679] Signal: Segmentation fault (11) +[b11n13:153679] Signal code: (3) +[b11n13:153679] Failing at address: 0xb170000000000a0 +[b11n13:153679] [ 0] [0x1000000504d8] +[b11n13:153679] [ 1] [0x3fffc62a69c0] +[b11n13:153679] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n13:153679] [ 3] [b16n17:119978] *** Process received signal *** +[b16n17:119978] Signal: Segmentation fault (11) +[b16n17:119978] Signal code: (3) +[b16n17:119978] Failing at address: 0x14170000000000a0 +[b16n17:119978] [ 0] [0x1000000504d8] +[b16n17:119978] [ 1] [0x3fffe5c15e90] +[b16n17:119978] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n17:119978] [ 3] [b05n18:165697] *** Process received signal *** +[b05n18:165697] Signal: Segmentation fault (11) +[b05n18:165697] Signal code: (3) +[b05n18:165697] Failing at address: 0xb170000000000a0 +[b05n18:165697] [ 0] [0x1000000504d8] +[b05n18:165697] [ 1] [0x3fffc9b7d380] +[b05n18:165697] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n18:165697] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n18:165697] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n18:165697] [ 5] [b23n08:116901] *** Process received signal *** +[b23n08:116901] Signal: Segmentation fault (11) +[b23n08:116901] Signal code: (3) +[b23n08:116901] Failing at address: 0xb170000000000a0 +[b23n08:116901] [ 0] [0x1000000504d8] +[b23n08:116901] [ 1] [0x3fffc831ef20] +[b23n08:116901] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n08:116901] [ 3] [b22n04:44756] *** Process received signal *** +[b22n04:44756] Signal: Segmentation fault (11) +[b22n04:44756] Signal code: (3) +[b22n04:44756] Failing at address: 0xb170000000000a0 +[b22n04:44756] [ 0] [0x1000000504d8] +[b22n04:44756] [ 1] [0x3ffff0e00380] +[b22n04:44756] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n04:44756] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n11:03987] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n11:03987] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n11:03987] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n11:03987] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n11:03987] [ 8] [0x3ffff8d018d0] +[a35n16:148189] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n16:148189] [ 3] [b22n03:120671] *** Process received signal *** +[b22n03:120671] Signal: Segmentation fault (11) +[b22n03:120671] Signal code: (3) +[b22n03:120671] Failing at address: 0x14170000000000a0 +[b22n03:120671] [ 0] [0x1000000504d8] +[b22n03:120671] [ 1] [0x3fffcd8367c0] +[b22n03:120671] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n03:120671] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n03:120671] [ 4] [b04n16:08385] *** Process received signal *** +[b04n16:08385] Signal: Segmentation fault (11) +[b04n16:08385] Signal code: (3) +[b04n16:08385] Failing at address: 0x14170000000000a0 +[b04n16:08385] [ 0] [0x1000000504d8] +[b04n16:08385] [ 1] [0x3ffffa4c2710] +[b04n16:08385] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n16:08385] [ 3] [a36n05:139910] Signal: Segmentation fault (11) +[a36n05:139910] Signal code: (3) +[a36n05:139910] Failing at address: 0xb170000000000a0 +[a36n05:139910] [ 0] [0x1000000504d8] +[a36n05:139910] [ 1] [0x3fffeabd9ef0] +[a36n05:139910] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n18:179168] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n18:179168] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n18:179168] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n18:179168] [ 7] [b14n12:123559] *** Process received signal *** +[b14n12:123559] Signal: Segmentation fault (11) +[b14n12:123559] Signal code: (3) +[b14n12:123559] Failing at address: 0xb170000000000a0 +[b14n12:123559] [ 0] [0x1000000504d8] +[b14n12:123559] [ 1] [0x3ffff1e5d950] +[b14n12:123559] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n12:123559] [ 3] [b18n13:123054] *** Process received signal *** +[b18n13:123054] Signal: Segmentation fault (11) +[b18n13:123054] Signal code: (3) +[b18n13:123054] Failing at address: 0x14170000000000a0 +[b18n13:123054] [ 0] [0x1000000504d8] +[b18n13:123054] [ 1] [0x3fffe744bf60] +[b18n13:123054] [ 2] [0x3fffc3656e60] +[b12n10:140991] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n10:140991] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n10:140991] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n10:140991] [ 5] [a29n08:159021] *** Process received signal *** +[a29n08:159021] Signal: Segmentation fault (11) +[a29n08:159021] Signal code: (3) +[a29n08:159021] Failing at address: 0xb170000000000a0 +[a29n08:159021] [ 0] [0x1000000504d8] +[a29n08:159021] [ 1] [0x3fffd0fd0750] +[a29n08:159021] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n08:159021] [ 3] [b14n11:130806] *** Process received signal *** +[b14n11:130806] Signal: Segmentation fault (11) +[b14n11:130806] Signal code: (3) +[b14n11:130806] Failing at address: 0x14170000000000a0 +[b14n11:130806] [ 0] [0x1000000504d8] +[b14n11:130806] [ 1] [0x3fffc247c840] +[b14n11:130806] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n11:130806] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n18:123413] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n18:123413] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n05:131264] *** End of error message *** +[a33n06:165304] *** Process received signal *** +[a33n06:165304] Signal: Segmentation fault (11) +[a33n06:165304] Signal code: (3) +[a33n06:165304] Failing at address: 0x14170000000000a0 +[a33n06:165304] [ 0] [0x1000000504d8] +[a33n06:165304] [ 1] [0x3ffff80d2380] +[a33n06:165304] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n06:165304] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n06:165304] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n06:165304] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n06:165304] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n06:165304] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n06:165304] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n06:165304] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n06:165304] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n06:165304] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n06:165304] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n06:165304] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n06:165304] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n06:165304] *** End of error message *** +[a28n05:04521] *** Process received signal *** +[a28n05:04521] Signal: Segmentation fault (11) +[a28n05:04521] Signal code: (3) +[a28n05:04521] Failing at address: 0x14170000000000a0 +[a28n05:04521] [ 0] [0x1000000504d8] +[a28n05:04521] [ 1] [b11n16:154185] *** Process received signal *** +[b11n16:154185] Signal: Segmentation fault (11) +[b11n16:154185] Signal code: (3) +[b11n16:154185] Failing at address: 0xb170000000000a0 +[b11n16:154185] [ 0] [0x1000000504d8] +[b11n16:154185] [ 1] [0x3fffe93cc870] +[b11n16:154185] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n16:154185] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n16:154185] [ 4] [a34n12:145500] *** Process received signal *** +[a34n12:145500] Signal: Segmentation fault (11) +[a34n12:145500] Signal code: (3) +[a34n12:145500] Failing at address: 0x14170000000000a0 +[a34n12:145500] [ 0] [0x1000000504d8] +[a34n12:145500] [ 1] [0x3fffe88b4380] +[a34n12:145500] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n12:145500] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n12:145500] [ 4] [b20n10:131275] *** Process received signal *** +[b20n10:131275] Signal: Segmentation fault (11) +[b20n10:131275] Signal code: (3) +[b20n10:131275] Failing at address: 0xb170000000000a0 +[b20n10:131275] [ 0] [0x1000000504d8] +[b20n10:131275] [ 1] [0x3fffd3fe4ac0] +[b20n10:131275] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n10:131275] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n10:131275] [ 4] [b12n07:140136] *** Process received signal *** +[b12n07:140136] Signal: Segmentation fault (11) +[b12n07:140136] Signal code: (3) +[b12n07:140136] Failing at address: 0x14170000000000a0 +[b12n07:140136] [ 0] [0x1000000504d8] +[b12n07:140136] [ 1] [0x3fffd8937d50] +[b12n07:140136] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n07:140136] [ 3] [b18n15:120133] *** Process received signal *** +[b18n15:120133] Signal: Segmentation fault (11) +[b18n15:120133] Signal code: (3) +[b18n15:120133] Failing at address: 0x14170000000000a0 +[b18n15:120133] [ 0] [b05n14:177509] *** Process received signal *** +[b05n14:177509] Signal: Segmentation fault (11) +[b05n14:177509] Signal code: (3) +[b05n14:177509] Failing at address: 0xb170000000000a0 +[b05n14:177509] [ 0] [0x1000000504d8] +[b05n14:177509] [ 1] [0x3fffc791dc00] +[b05n14:177509] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n14:177509] [ 3] [b23n05:120230] *** Process received signal *** +[b23n05:120230] Signal: Segmentation fault (11) +[b23n05:120230] Signal code: (3) +[b23n05:120230] Failing at address: 0x14170000000000a0 +[b23n05:120230] [ 0] [0x1000000504d8] +[b23n05:120230] [ 1] [0x3fffe3f764b0] +[b23n05:120230] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n05:120230] [ 3] [b17n08:124116] *** Process received signal *** +[b17n08:124116] Signal: Segmentation fault (11) +[b17n08:124116] Signal code: (3) +[b17n08:124116] Failing at address: 0xb170000000000a0 +[b17n08:124116] [ 0] [0x1000000504d8] +[b17n08:124116] [ 1] [0x3ffff91eb770] +[b17n08:124116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n08:124116] [ 3] [b09n09:170657] *** Process received signal *** +[b09n09:170657] Signal: Segmentation fault (11) +[b09n09:170657] Signal code: (3) +[b09n09:170657] Failing at address: 0x14170000000000a0 +[b09n09:170657] [ 0] [0x1000000504d8] +[b09n09:170657] [ 1] [0x3fffe3911d20] +[b09n09:170657] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n09:170657] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n10:122761] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n10:122761] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n10:122761] [ 9] [b20n11:123522] *** Process received signal *** +[b20n11:123522] Signal: Segmentation fault (11) +[b20n11:123522] Signal code: (3) +[b20n11:123522] Failing at address: 0xb170000000000a0 +[b20n11:123522] [ 0] [0x1000000504d8] +[b20n11:123522] [ 1] [0x3fffedb2b470] +[b20n11:123522] [ 2] [0x3fffc1f05d00] +[a34n02:139630] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n02:139630] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n02:139630] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n02:139630] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n14:126120] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n14:126120] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n14:126120] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n16:124962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n16:124962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n16:124962] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n16:124962] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n16:124962] [14] [a29n12:160189] *** Process received signal *** +[a29n12:160189] Signal: Segmentation fault (11) +[a29n12:160189] Signal code: (3) +[a29n12:160189] Failing at address: 0x14170000000000a0 +[a29n12:160189] [ 0] [0x1000000504d8] +[a29n12:160189] [ 1] [0x3fffd5913c70] +[a29n12:160189] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n12:160189] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n04:06833] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n04:06833] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n04:06833] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n04:06833] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n04:06833] [10] [b20n04:126349] *** Process received signal *** +[b20n04:126349] Signal: Segmentation fault (11) +[b20n04:126349] Signal code: (3) +[b20n04:126349] Failing at address: 0x14170000000000a0 +[b20n04:126349] [a33n12:168644] *** Process received signal *** +[a33n12:168644] Signal: Segmentation fault (11) +[a33n12:168644] Signal code: (3) +[a33n12:168644] Failing at address: 0xb170000000000a0 +[a33n12:168644] [ 0] [0x1000000504d8] +[a33n12:168644] [ 1] [0x3fffe4f27880] +[a33n12:168644] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n12:168644] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n07:116114] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n07:116114] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n07:116114] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n07:116114] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n07:116114] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n14:116415] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n14:116415] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n14:116415] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n14:116415] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n02:149229] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n02:149229] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n02:149229] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n02:149229] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n02:149229] *** End of error message *** +[0x3fffc150f5a0] +[a27n05:53376] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n05:53376] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n05:53376] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n05:53376] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n05:53376] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n05:53376] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n05:53376] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n05:53376] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n05:53376] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n05:53376] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n05:53376] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n05:53376] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n11:23064] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n07:156144] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n07:156144] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n07:156144] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n07:156144] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n18:121303] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n18:121303] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n18:121303] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n16:147288] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n16:147288] [11] [b08n04:172230] *** Process received signal *** +[b08n04:172230] Signal: Segmentation fault (11) +[b08n04:172230] Signal code: (3) +[b08n04:172230] Failing at address: 0xb170000000000a0 +[b08n04:172230] [ 0] [0x1000000504d8] +[b08n04:172230] [ 1] [0x3fffdfa842c0] +[b08n04:172230] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n04:172230] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n04:172230] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n04:172230] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n04:172230] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n04:172230] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n04:172230] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n04:172230] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n04:172230] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n04:172230] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n04:172230] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n04:172230] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n04:172230] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n04:172230] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n17:08307] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n17:08307] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n17:08307] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n17:08307] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n17:08307] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n17:08307] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n17:08307] [ 9] [a28n12:177496] *** Process received signal *** +[a28n12:177496] Signal: Segmentation fault (11) +[a28n12:177496] Signal code: (3) +[a28n12:177496] Failing at address: 0x14170000000000a0 +[a28n12:177496] [ 0] [0x1000000504d8] +[a28n12:177496] [ 1] [0x3fffff935330] +[a28n12:177496] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n12:177496] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n12:177496] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n04:177596] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n04:177596] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n04:177596] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n17:15872] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n17:15872] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n17:15872] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n17:15872] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n17:15872] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n17:15872] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n17:15872] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n17:15872] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n17:15872] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n17:15872] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n03:136666] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n03:136666] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n16:168050] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n16:168050] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n18:03315] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n18:03315] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n18:03315] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n18:03315] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n18:03315] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n18:03315] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n18:03315] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n18:03315] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n18:03315] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n18:03315] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n09:123420] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n09:123420] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n09:123420] [14] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n04:117354] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n04:117354] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n13:131759] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n13:131759] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n13:131759] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n13:131759] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n13:131759] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n13:131759] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n09:62481] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n09:62481] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n09:62481] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n09:62481] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n09:62481] *** End of error message *** +[b13n09:143898] *** Process received signal *** +[b13n09:143898] Signal: Segmentation fault (11) +[b13n09:143898] Signal code: (3) +[b13n09:143898] Failing at address: 0xb170000000000a0 +[b13n09:143898] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n14:139845] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n14:139845] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n14:139845] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n14:139845] [ 8] [b09n10:169685] *** Process received signal *** +[b09n10:169685] Signal: Segmentation fault (11) +[b09n10:169685] Signal code: (3) +[b09n10:169685] Failing at address: 0xb170000000000a0 +[b09n10:169685] [ 0] [0x1000000504d8] +[b09n10:169685] [ 1] [0x3ffff7e94d80] +[b09n10:169685] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n10:169685] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n10:169685] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n13:130059] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n13:130059] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n13:130059] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n13:130059] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n13:130059] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n13:130059] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n13:130059] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n13:130059] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n13:130059] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n13:130059] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n13:130059] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n13:130059] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n13:130059] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n07:149478] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n07:149478] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n07:149478] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n07:149478] [ 7] [b20n12:119805] *** Process received signal *** +[b20n12:119805] Signal: Segmentation fault (11) +[b20n12:119805] Signal code: (3) +[b20n12:119805] Failing at address: 0xb170000000000a0 +[b20n12:119805] [ 0] [b20n14:120638] [ 0] [0x1000000504d8] +[b20n14:120638] [ 1] [0x3fffca0f1120] +[b20n14:120638] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n14:120638] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n13:177641] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n13:177641] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n13:177641] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n13:177641] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n13:177641] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n08:144853] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n08:144853] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n13:122188] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n13:122188] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n10:126192] *** End of error message *** +[b16n09:124193] *** Process received signal *** +[b16n09:124193] Signal: Segmentation fault (11) +[b16n09:124193] Signal code: (3) +[b16n09:124193] Failing at address: 0x14170000000000a0 +[b16n09:124193] [ 0] [0x1000000504d8] +[b16n09:124193] [ 1] [0x3fffd7f5ea40] +[b16n09:124193] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n09:124193] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n09:124193] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n09:124193] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n15:07537] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n15:07537] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n15:07537] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n15:07537] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n15:07537] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n15:07537] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n15:07537] *** End of error message *** +[b17n17:125809] *** Process received signal *** +[b17n17:125809] Signal: Segmentation fault (11) +[b17n17:125809] Signal code: (3) +[b17n17:125809] Failing at address: 0x14170000000000a0 +[b06n07:174562] *** Process received signal *** +[b06n07:174562] Signal: Segmentation fault (11) +[b06n07:174562] Signal code: (3) +[b06n07:174562] Failing at address: 0x14170000000000a0 +[b06n07:174562] [ 0] [0x1000000504d8] +[b06n07:174562] [ 1] [0x3fffd2e89540] +[b06n07:174562] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n07:174562] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n07:174562] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n12:180116] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n12:180116] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n12:180116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n07:173042] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n07:173042] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n07:173042] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n13:138089] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n13:138089] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n13:138089] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n13:138089] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n15:32401] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n07:03373] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n07:03373] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n12:164381] [ 6] [b09n01:163025] *** Process received signal *** +[b09n01:163025] Signal: Segmentation fault (11) +[b09n01:163025] Signal code: (3) +[b09n01:163025] Failing at address: 0xb170000000000a0 +[b09n01:163025] [ 0] [0x1000000504d8] +[b09n01:163025] [ 1] [0x3fffdadee930] +[b09n01:163025] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n01:163025] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n01:163025] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n01:163025] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n18:176811] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n18:176811] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n02:167597] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n02:167597] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n02:167597] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n02:167597] [10] [b06n13:166926] *** Process received signal *** +[b06n13:166926] Signal: Segmentation fault (11) +[b06n13:166926] Signal code: (3) +[b06n13:166926] Failing at address: 0x14170000000000a0 +[b06n13:166926] [ 0] [0x1000000504d8] +[b06n13:166926] [ 1] [0x3fffc73f9e30] +[b06n13:166926] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n13:166926] [ 3] [b11n06:164016] *** End of error message *** +[b09n18:164796] *** Process received signal *** +[b09n18:164796] Signal: Segmentation fault (11) +[b09n18:164796] Signal code: (3) +[b09n18:164796] Failing at address: 0x14170000000000a0 +[b09n18:164796] [ 0] [b04n02:177547] *** Process received signal *** +[b04n02:177547] Signal: Segmentation fault (11) +[b04n02:177547] Signal code: (3) +[b04n02:177547] Failing at address: 0xb170000000000a0 +[b04n02:177547] [ 0] [0x1000000504d8] +[b04n02:177547] [ 1] [0x3fffe38aaf60] +[b04n02:177547] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n02:177547] [ 3] [b06n18:173978] *** Process received signal *** +[b06n18:173978] Signal: Segmentation fault (11) +[b06n18:173978] Signal code: (3) +[b06n18:173978] Failing at address: 0x14170000000000a0 +[b06n18:173978] [ 0] [0x1000000504d8] +[b06n18:173978] [ 1] [0x3ffff8482fb0] +[b06n18:173978] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n18:173978] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n18:173978] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n15:119251] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n15:119251] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n15:119251] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n15:119251] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n14:119485] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n14:119485] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n11:162471] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n11:162471] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n14:150132] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n07:08397] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n07:08397] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n07:08397] [ 6] [b23n10:126788] *** Process received signal *** +[b23n10:126788] Signal: Segmentation fault (11) +[b23n10:126788] Signal code: (3) +[b23n10:126788] Failing at address: 0xb170000000000a0 +[b23n10:126788] [ 0] [0x1000000504d8] +[b23n10:126788] [ 1] [0x3fffc5a96140] +[b23n10:126788] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n10:126788] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n10:126788] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n10:126788] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n10:126788] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n10:126788] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n10:126788] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n10:126788] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n10:126788] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n10:126788] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n10:126788] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n10:126788] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n10:126788] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n10:126788] *** End of error message *** +[0x1000000504d8] +[b21n13:117511] [ 1] [0x3ffffc950f70] +[b21n13:117511] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n13:117511] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n13:117511] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n13:117511] [ 5] [b03n02:06561] Signal: Segmentation fault (11) +[b03n02:06561] Signal code: (3) +[b03n02:06561] Failing at address: 0x14170000000000a0 +[b03n02:06561] [ 0] [0x1000000504d8] +[b03n02:06561] [ 1] [0x3fffcd028d00] +[b03n02:06561] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n15:154384] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n17:03845] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n17:03845] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n17:03845] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n17:03845] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n17:03845] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n17:03845] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n10:134735] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n10:134735] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n10:134735] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n10:134735] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n10:134735] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n16:173548] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n16:173548] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n16:173548] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n18:08429] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n18:08429] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n18:08429] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n18:08429] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n18:08429] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n11:163262] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n11:163262] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n11:163262] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n11:163262] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n11:160797] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n11:160797] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n11:160797] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n11:160797] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n11:160797] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n11:160797] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n05:154190] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n05:154190] *** End of error message *** +[a32n04:153965] *** Process received signal *** +[a32n04:153965] Signal: Segmentation fault (11) +[a32n04:153965] Signal code: (3) +[a32n04:153965] Failing at address: 0x5170000000000a0 +[a32n04:153965] [ 0] [0x1000000504d8] +[a32n04:153965] [ 1] [0x3ffff3f02e80] +[a32n04:153965] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n04:153965] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n14:21318] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n03:158660] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n03:158660] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n03:158660] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n06:135850] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n06:135850] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n06:135850] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n06:135850] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n18:177222] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n18:177222] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n18:177222] [ 9] [b21n12:120131] *** Process received signal *** +[b21n12:120131] Signal: Segmentation fault (11) +[b21n12:120131] Signal code: (3) +[b21n12:120131] Failing at address: 0x14170000000000a0 +[b21n12:120131] [ 0] [0x1000000504d8] +[b21n12:120131] [ 1] [0x3fffcefa5290] +[b21n12:120131] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n12:120131] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n11:153420] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n11:153420] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n11:153420] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n10:173683] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n10:173683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n10:173683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n10:173683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n16:149064] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n16:149064] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n16:149064] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n16:149064] [14] [b03n06:01838] *** Process received signal *** +[b03n06:01838] Signal: Segmentation fault (11) +[b03n06:01838] Signal code: (3) +[b03n06:01838] Failing at address: 0x14170000000000a0 +[b03n06:01838] [ 0] [0x1000000504d8] +[b03n06:01838] [ 1] [0x3fffe10a4fc0] +[b03n06:01838] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n06:01838] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n06:01838] [ 4] [b12n09:127999] *** Process received signal *** +[b12n09:127999] Signal: Segmentation fault (11) +[b12n09:127999] Signal code: (3) +[b12n09:127999] Failing at address: 0xb170000000000a0 +[b12n09:127999] [ 0] [0x1000000504d8] +[b12n09:127999] [ 1] [0x3ffffc09e920] +[b12n09:127999] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n09:127999] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n18:138475] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n18:138475] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n18:138475] [10] [b15n03:119829] *** Process received signal *** +[b15n03:119829] Signal: Segmentation fault (11) +[b15n03:119829] Signal code: (3) +[b15n03:119829] Failing at address: 0x14170000000000a0 +[b15n03:119829] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n07:133377] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n07:133377] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n07:133377] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n07:133377] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n07:133377] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n07:133377] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n17:171201] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n17:171201] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n17:171201] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n17:171201] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n17:171201] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n17:171201] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n17:171201] *** End of error message *** +[0x1000000504d8] +[b21n08:116171] [ 1] [0x3fffe7e5ca60] +[b21n08:116171] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n08:116171] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n08:116171] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n16:165921] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n04:52704] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n04:52704] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n05:54536] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n05:54536] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n05:54536] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n05:54536] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n18:07424] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n18:07424] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n18:07424] *** End of error message *** +[ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n07:117359] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n16:127354] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n16:127354] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n16:127354] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n16:127354] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n16:127354] *** End of error message *** +[b11n04:166207] *** Process received signal *** +[b11n04:166207] Signal: Segmentation fault (11) +[b11n04:166207] Signal code: (3) +[b11n04:166207] Failing at address: 0xb170000000000a0 +[b11n04:166207] [ 0] [0x1000000504d8] +[b11n04:166207] [ 1] [0x3fffcb3b1150] +[b11n04:166207] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n04:166207] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n04:166207] [ 4] [b18n17:124166] *** Process received signal *** +[b18n17:124166] Signal: Segmentation fault (11) +[b18n17:124166] Signal code: (3) +[b18n17:124166] Failing at address: 0x14170000000000a0 +[b18n17:124166] [ 0] [0x1000000504d8] +[b18n17:124166] [ 1] [0x3fffc3f459b0] +[b18n17:124166] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n17:124166] [ 3] [b22n13:122033] *** Process received signal *** +[b22n13:122033] Signal: Segmentation fault (11) +[b22n13:122033] Signal code: (3) +[b22n13:122033] Failing at address: 0x14170000000000a0 +[b22n13:122033] [ 0] [0x1000000504d8] +[b22n13:122033] [ 1] [0x3fffdbd44aa0] +[b22n13:122033] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n13:122033] [ 3] [0x3fffecbfbbb0] +[b16n11:124434] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n11:124434] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n11:124434] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n11:124434] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n16:128443] [ 6] [0x1000000504d8] +[a26n18:69156] [ 1] [0x3fffca48cce0] +[a26n18:69156] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n18:69156] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n18:69156] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n10:124501] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n10:124501] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n10:124501] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n10:124501] [10] [b03n14:177224] *** Process received signal *** +[b03n14:177224] Signal: Segmentation fault (11) +[b03n14:177224] Signal code: (3) +[b03n14:177224] Failing at address: 0x14170000000000a0 +[b03n14:177224] [ 0] [0x1000000504d8] +[b03n14:177224] [ 1] [0x3fffc505ae10] +[b03n14:177224] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n14:177224] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n14:177224] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n14:177224] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n14:08952] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n14:08952] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n14:08952] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n14:08952] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n03:38501] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n03:38501] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n03:38501] [ 8] [a27n01:48806] *** Process received signal *** +[a27n01:48806] Signal: Segmentation fault (11) +[a27n01:48806] Signal code: (3) +[a27n01:48806] Failing at address: 0x14170000000000a0 +[a27n01:48806] [ 0] [b08n12:12761] Signal: Segmentation fault (11) +[b08n12:12761] Signal code: (3) +[b08n12:12761] Failing at address: 0x14170000000000a0 +[b08n12:12761] [ 0] [0x1000000504d8] +[b08n12:12761] [ 1] [0x3fffee8f6100] +[b08n12:12761] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n12:12761] [ 3] [b15n12:132099] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n13:114397] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n13:114397] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n12:167677] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n12:167677] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n12:167677] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n12:167677] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n12:167677] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n12:167677] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n15:177059] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n15:177059] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n14:161616] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n14:161616] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n14:161616] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n14:161616] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n14:161616] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n14:161616] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n14:161616] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n14:161616] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n14:161616] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n14:161616] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n14:161616] *** End of error message *** +[a36n10:142930] *** Process received signal *** +[a36n10:142930] Signal: Segmentation fault (11) +[a36n10:142930] Signal code: (3) +[a36n10:142930] Failing at address: 0xb170000000000a0 +[a36n10:142930] [ 0] [0x1000000504d8] +[a36n10:142930] [ 1] [0x3fffd665ac20] +[a36n10:142930] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n10:142930] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n11:160351] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n11:160351] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n11:160351] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n11:160351] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n17:149905] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n17:149905] *** End of error message *** +[0x1000000504d8] +[b21n16:116525] [ 1] [0x3fffe58d78a0] +[b21n16:116525] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n16:116525] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n16:116525] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n16:116525] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n05:135164] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n16:157888] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n16:157888] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n16:157888] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n03:177924] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n03:177924] *** End of error message *** +[b23n07:118166] *** Process received signal *** +[b23n07:118166] Signal: Segmentation fault (11) +[b23n07:118166] Signal code: (3) +[b23n07:118166] Failing at address: 0x14170000000000a0 +[b23n07:118166] [ 0] [0x1000000504d8] +[b23n07:118166] [ 1] [0x3fffd94c6480] +[b23n07:118166] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n07:118166] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n03:123619] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n03:123619] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n03:123619] [ 8] [b18n14:128870] *** Process received signal *** +[b18n14:128870] Signal: Segmentation fault (11) +[b18n14:128870] Signal code: (3) +[b18n14:128870] Failing at address: 0x14170000000000a0 +[b18n14:128870] [ 0] [0x1000000504d8] +[b18n14:128870] [ 1] [0x3fffd7f0bd90] +[b18n14:128870] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n14:128870] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n11:62421] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n11:62421] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n09:03928] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n09:03928] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n10:176311] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n10:176311] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n10:176311] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n10:176311] [ 8] [b16n14:133632] *** Process received signal *** +[b16n14:133632] Signal: Segmentation fault (11) +[b16n14:133632] Signal code: (3) +[b16n14:133632] Failing at address: 0x14170000000000a0 +[b16n14:133632] [ 0] [0x1000000504d8] +[b16n14:133632] [ 1] [b02n14:169522] *** Process received signal *** +[b02n14:169522] Signal: Segmentation fault (11) +[b02n14:169522] Signal code: (3) +[b02n14:169522] Failing at address: 0xb170000000000a0 +[b02n14:169522] [b01n18:179700] *** Process received signal *** +[b01n18:179700] Signal: Segmentation fault (11) +[b01n18:179700] Signal code: (3) +[b01n18:179700] Failing at address: 0x5170000000000a0 +[b01n18:179700] [ 0] [0x1000000504d8] +[b01n18:179700] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n14:173894] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n14:173894] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n01:141816] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n01:141816] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n01:141816] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n01:141816] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n11:171963] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n11:171963] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n11:171963] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n11:171963] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n16:124664] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n16:124664] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n06:143012] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n06:143012] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n06:143012] [ 6] [0x1000000504d8] +[a33n08:147824] [ 1] [0x3fffe25fccf0] +[a33n08:147824] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n08:147824] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n08:147824] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n08:147824] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n08:147824] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n08:147824] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n08:147824] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n08:08957] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n08:08957] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n18:126908] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n18:126908] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n18:126908] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n18:126908] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n07:132089] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n07:132089] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n07:132089] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n04:169911] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n04:169911] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n04:169911] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n04:169911] [10] [a26n10:67857] *** Process received signal *** +[a26n10:67857] Signal: Segmentation fault (11) +[a26n10:67857] Signal code: (3) +[a26n10:67857] Failing at address: 0xb170000000000a0 +[a26n10:67857] [ 0] [0x1000000504d8] +[a26n10:67857] [ 1] [0x3fffe914aa50] +[a26n10:67857] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n02:164493] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n02:164493] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n02:164493] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n02:164493] [14] [a36n07:153178] *** Process received signal *** +[a36n07:153178] Signal: Segmentation fault (11) +[a36n07:153178] Signal code: (3) +[a36n07:153178] Failing at address: 0x5170000000000a0 +[a36n07:153178] [ 0] [0x1000000504d8] +[a36n07:153178] [ 1] [0x3fffcbfe4580] +[a36n07:153178] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n07:153178] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n07:153178] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n13:156921] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n13:156921] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n13:123707] *** End of error message *** +[b11n11:152986] *** Process received signal *** +[b11n11:152986] Signal: Segmentation fault (11) +[b11n11:152986] Signal code: (3) +[b11n11:152986] Failing at address: 0xb170000000000a0 +[b11n11:152986] [ 0] [0x1000000504d8] +[b11n11:152986] [ 1] [0x3fffc74738b0] +[b11n11:152986] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n11:152986] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n04:126859] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n04:126859] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n18:28221] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n18:28221] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n18:28221] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n18:28221] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n18:28221] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n15:85940] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n15:85940] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n15:85940] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n15:85940] [14] [b05n12:11141] *** Process received signal *** +[b05n12:11141] Signal: Segmentation fault (11) +[b05n12:11141] Signal code: (3) +[b05n12:11141] Failing at address: 0x14170000000000a0 +[b05n12:11141] [ 0] [0x1000000504d8] +[b05n12:11141] [ 1] [0x3fffd727bdd0] +[b05n12:11141] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n12:11141] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n16:01573] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n16:01573] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n16:01573] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n09:173750] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n09:173750] [10] [a28n03:10292] *** Process received signal *** +[a28n03:10292] Signal: Segmentation fault (11) +[a28n03:10292] Signal code: (3) +[a28n03:10292] Failing at address: 0x14170000000000a0 +[a28n03:10292] [ 0] [0x1000000504d8] +[a28n03:10292] [ 1] [b15n09:121755] *** Process received signal *** +[b15n09:121755] Signal: Segmentation fault (11) +[b15n09:121755] Signal code: (3) +[b15n09:121755] Failing at address: 0x14170000000000a0 +[b15n09:121755] [ 0] [0x1000000504d8] +[b15n09:121755] [ 1] [0x3fffe0f4d690] +[b15n09:121755] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n09:121755] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n10:130583] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n10:130583] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n14:168376] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n14:168376] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n11:01170] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n11:01170] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n08:124725] [ 5] [b14n09:119549] *** Process received signal *** +[b14n09:119549] Signal: Segmentation fault (11) +[b14n09:119549] Signal code: (3) +[b14n09:119549] Failing at address: 0x14170000000000a0 +[b14n09:119549] [ 0] [0x1000000504d8] +[b14n09:119549] [ 1] [0x3fffd4663380] +[b14n09:119549] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n09:119549] [ 3] [b03n13:01300] *** Process received signal *** +[b03n13:01300] Signal: Segmentation fault (11) +[b03n13:01300] Signal code: (3) +[b03n13:01300] Failing at address: 0x5170000000000a0 +[b03n13:01300] [ 0] [b12n11:137874] *** Process received signal *** +[b12n11:137874] Signal: Segmentation fault (11) +[b12n11:137874] Signal code: (3) +[b12n11:137874] Failing at address: 0x5170000000000a0 +[b12n11:137874] [ 0] [0x1000000504d8] +[b12n11:137874] [ 1] [0x3fffeb909050] +[b12n11:137874] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n11:137874] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n16:153290] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n16:153290] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n16:153290] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n16:153290] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n16:171918] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n16:171918] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n15:177862] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n15:177862] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n15:177862] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n15:177862] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n15:177862] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n17:150932] [14] [b06n08:174412] *** Process received signal *** +[b06n08:174412] Signal: Segmentation fault (11) +[b06n08:174412] Signal code: (3) +[b06n08:174412] Failing at address: 0x14170000000000a0 +[b06n08:174412] [ 0] [0x1000000504d8] +[b06n08:174412] [ 1] [0x3fffc34a82d0] +[b06n08:174412] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n08:174412] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n08:174412] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n08:174412] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n08:174412] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n15:55851] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n15:55851] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n15:55851] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n15:55851] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n15:55851] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n15:55851] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n15:55851] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n15:55851] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n15:55851] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n15:55851] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n15:55851] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n15:55851] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n15:55851] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n11:152843] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n11:152843] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n11:152843] [ 6] [a33n15:148580] *** Process received signal *** +[a33n15:148580] Signal: Segmentation fault (11) +[a33n15:148580] Signal code: (3) +[a33n15:148580] Failing at address: 0xb170000000000a0 +[a33n15:148580] [ 0] [0x1000000504d8] +[a33n15:148580] [ 1] [b06n04:175999] *** Process received signal *** +[b06n04:175999] Signal: Segmentation fault (11) +[b06n04:175999] Signal code: (3) +[b06n04:175999] Failing at address: 0x14170000000000a0 +[b06n04:175999] [ 0] [0x1000000504d8] +[b06n04:175999] [ 1] [0x3ffffaf9d2f0] +[b06n04:175999] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n04:175999] [ 3] [b07n10:16975] *** Process received signal *** +[b07n10:16975] Signal: Segmentation fault (11) +[b07n10:16975] Signal code: (3) +[b07n10:16975] Failing at address: 0x14170000000000a0 +[b07n10:16975] [ 0] [0x1000000504d8] +[b07n10:16975] [ 1] [0x3fffdb5e7a80] +[b07n10:16975] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n10:16975] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n10:16975] [ 4] [b04n06:10224] *** Process received signal *** +[b04n06:10224] Signal: Segmentation fault (11) +[b04n06:10224] Signal code: (3) +[b04n06:10224] Failing at address: 0xb170000000000a0 +[b11n12:154341] *** Process received signal *** +[b11n12:154341] Signal: Segmentation fault (11) +[b11n12:154341] Signal code: (3) +[b11n12:154341] Failing at address: 0x14170000000000a0 +[b11n12:154341] [ 0] [0x1000000504d8] +[b11n12:154341] [ 1] [b23n12:115637] *** Process received signal *** +[b23n12:115637] Signal: Segmentation fault (11) +[b23n12:115637] Signal code: (3) +[b23n12:115637] Failing at address: 0xb170000000000a0 +[b23n12:115637] [ 0] [b22n11:120242] *** Process received signal *** +[b22n11:120242] Signal: Segmentation fault (11) +[b22n11:120242] Signal code: (3) +[b22n11:120242] Failing at address: 0x14170000000000a0 +[b04n05:169787] *** Process received signal *** +[b04n05:169787] Signal: Segmentation fault (11) +[b04n05:169787] Signal code: (3) +[b04n05:169787] Failing at address: 0xb170000000000a0 +[b04n05:169787] [ 0] [0x1000000504d8] +[b04n05:169787] [ 1] [0x3fffe48ea9e0] +[b04n05:169787] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n05:169787] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n05:169787] [ 4] [b16n18:134183] *** Process received signal *** +[b16n18:134183] Signal: Segmentation fault (11) +[b16n18:134183] Signal code: (3) +[b16n18:134183] Failing at address: 0x14170000000000a0 +[b16n18:134183] [ 0] [0x1000000504d8] +[b16n18:134183] [ 1] [0x3ffff2a42c10] +[b16n18:134183] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n18:134183] [ 3] [b05n06:177848] *** Process received signal *** +[b05n06:177848] Signal: Segmentation fault (11) +[b05n06:177848] Signal code: (3) +[b05n06:177848] Failing at address: 0xb170000000000a0 +[b05n06:177848] [ 0] [0x1000000504d8] +[b05n06:177848] [ 1] [0x3fffdb5d4230] +[b05n06:177848] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n06:177848] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n06:177848] [ 4] [b17n15:137007] *** Process received signal *** +[b17n15:137007] Signal: Segmentation fault (11) +[b17n15:137007] Signal code: (3) +[b17n15:137007] Failing at address: 0xb170000000000a0 +[b17n15:137007] [ 0] [b14n15:123040] *** Process received signal *** +[b14n15:123040] Signal: Segmentation fault (11) +[b14n15:123040] Signal code: (3) +[b14n15:123040] Failing at address: 0x14170000000000a0 +[b14n15:123040] [ 0] [0x1000000504d8] +[b14n15:123040] [ 1] [0x3fffd01ce400] +[b14n15:123040] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n15:123040] [ 3] [b20n09:121728] *** Process received signal *** +[b20n09:121728] Signal: Segmentation fault (11) +[b20n09:121728] Signal code: (3) +[b20n09:121728] Failing at address: 0xb170000000000a0 +[b20n09:121728] [ 0] [0x1000000504d8] +[b20n09:121728] [ 1] [0x3fffce237810] +[b20n09:121728] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n09:121728] [ 3] [a35n17:146798] *** Process received signal *** +[a35n17:146798] Signal: Segmentation fault (11) +[a35n17:146798] Signal code: (3) +[a35n17:146798] Failing at address: 0x5170000000000a0 +[a35n17:146798] [ 0] [a32n11:152814] *** Process received signal *** +[a32n11:152814] Signal: Segmentation fault (11) +[a32n11:152814] Signal code: (3) +[a32n11:152814] Failing at address: 0xb170000000000a0 +[b04n12:11420] *** Process received signal *** +[b04n12:11420] Signal: Segmentation fault (11) +[b04n12:11420] Signal code: (3) +[b04n12:11420] Failing at address: 0x14170000000000a0 +[b04n12:11420] [ 0] [0x1000000504d8] +[b04n12:11420] [ 1] [0x3fffc5125770] +[b04n12:11420] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n12:11420] [ 3] [b14n17:119468] *** Process received signal *** +[b14n17:119468] Signal: Segmentation fault (11) +[b14n17:119468] Signal code: (3) +[b14n17:119468] Failing at address: 0x14170000000000a0 +[b14n17:119468] [ 0] [b10n03:168929] *** Process received signal *** +[b10n03:168929] Signal: Segmentation fault (11) +[b10n03:168929] Signal code: (3) +[b10n03:168929] Failing at address: 0x14170000000000a0 +[b10n03:168929] [ 0] [0x1000000504d8] +[b10n03:168929] [ 1] [0x3fffc654dee0] +[b10n03:168929] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n03:168929] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n03:168929] [ 4] [b13n12:140120] *** Process received signal *** +[b13n12:140120] Signal: Segmentation fault (11) +[b13n12:140120] Signal code: (3) +[b13n12:140120] Failing at address: 0x14170000000000a0 +[b13n12:140120] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n02:49107] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n02:49107] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n11:02685] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n11:02685] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n11:02685] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n11:02685] *** End of error message *** +[b15n17:126527] *** Process received signal *** +[b15n17:126527] Signal: Segmentation fault (11) +[b15n17:126527] Signal code: (3) +[b15n17:126527] Failing at address: 0xb170000000000a0 +[b15n17:126527] [ 0] [0x1000000504d8] +[b15n17:126527] [ 1] [0x3fffe5dea1d0] +[b15n17:126527] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n17:126527] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n14:163548] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n14:163548] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n14:163548] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n14:163548] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n14:163548] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n14:163548] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n15:161755] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n15:161755] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n15:161755] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n13:06928] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n13:06928] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n13:06928] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n13:06928] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n13:06928] [14] [b06n17:163816] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n09:147795] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n09:147795] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n09:147795] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n09:147795] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n09:147795] [11] [b22n16:124797] *** Process received signal *** +[b22n16:124797] Signal: Segmentation fault (11) +[b22n16:124797] Signal code: (3) +[b22n16:124797] Failing at address: 0x14170000000000a0 +[b22n16:124797] [ 0] [0x1000000504d8] +[b22n16:124797] [ 1] [0x3ffff10edd00] +[b22n16:124797] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n16:124797] [ 3] [b23n03:125240] *** Process received signal *** +[b23n03:125240] Signal: Segmentation fault (11) +[b23n03:125240] Signal code: (3) +[b23n03:125240] Failing at address: 0x14170000000000a0 +[b23n03:125240] [ 0] [0x1000000504d8] +[b23n03:125240] [ 1] [0x3fffd8daae60] +[b23n03:125240] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n03:125240] [ 3] [0x1000000504d8] +[a32n06:154622] [ 1] [0x3ffff21a93d0] +[a32n06:154622] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n06:154622] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n06:154622] [ 4] [b11n05:162565] [ 0] [0x1000000504d8] +[b11n05:162565] [ 1] [0x3fffdd812b50] +[b11n05:162565] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n05:162565] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n05:162565] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n11:148673] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n11:148673] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n11:148673] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n11:148673] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n01:164962] *** End of error message *** +[b20n06:130543] *** Process received signal *** +[b20n06:130543] Signal: Segmentation fault (11) +[b20n06:130543] Signal code: (3) +[b20n06:130543] Failing at address: 0x14170000000000a0 +[b20n06:130543] [ 0] [0x1000000504d8] +[b20n06:130543] [ 1] [0x3ffff26995d0] +[b20n06:130543] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n06:130543] [ 3] [b01n08:172427] *** Process received signal *** +[b01n08:172427] Signal: Segmentation fault (11) +[b01n08:172427] Signal code: (3) +[b01n08:172427] Failing at address: 0x14170000000000a0 +[b01n08:172427] [ 0] [0x1000000504d8] +[b01n08:172427] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n02:122560] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n02:122560] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n03:151881] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n03:151881] [ 5] [b08n11:164997] *** Process received signal *** +[b08n11:164997] Signal: Segmentation fault (11) +[b08n11:164997] Signal code: (3) +[b08n11:164997] Failing at address: 0x14170000000000a0 +[b08n11:164997] [ 0] [0x1000000504d8] +[b08n11:164997] [ 1] [0x3fffde977410] +[b08n11:164997] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n11:164997] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n11:164997] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n11:164997] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n11:164997] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n11:164997] [ 7] [a28n08:175807] *** Process received signal *** +[a28n08:175807] Signal: Segmentation fault (11) +[a28n08:175807] Signal code: (3) +[a28n08:175807] Failing at address: 0x5170000000000a0 +[a28n08:175807] [ 0] [0x1000000504d8] +[a28n08:175807] [ 1] [0x3fffdef96b40] +[a28n08:175807] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n08:175807] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n10:161719] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n10:161719] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n10:161719] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n10:161719] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n10:161719] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n10:161719] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n10:161719] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n06:162919] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n06:162919] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n06:162919] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n05:173848] *** End of error message *** +[a29n10:164939] Signal: Segmentation fault (11) +[a29n10:164939] Signal code: (3) +[a29n10:164939] Failing at address: 0x14170000000000a0 +[a29n10:164939] [ 0] [0x1000000504d8] +[a29n10:164939] [ 1] [0x3fffc6a3b440] +[a29n10:164939] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n10:164939] [ 3] [0x3fffe397d030] +[b08n01:171784] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n01:171784] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n01:171784] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n01:171784] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n01:171784] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n01:171784] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n01:171784] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n01:171784] [ 9] [b05n09:106790] *** Process received signal *** +[b05n09:106790] Signal: Segmentation fault (11) +[b05n09:106790] Signal code: (3) +[b05n09:106790] Failing at address: 0x14170000000000a0 +[b05n09:106790] [ 0] [0x1000000504d8] +[b05n09:106790] [ 1] [0x3fffc86f43e0] +[b05n09:106790] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n09:106790] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n09:106790] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n09:106790] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n12:134493] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n12:134493] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n12:134493] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n12:134493] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n12:134493] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n12:134493] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n12:134493] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n12:134493] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n12:134493] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n05:120331] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n05:120331] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n05:120331] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n12:165844] [ 6] [0x1000000504d8] +[b08n08:164751] [ 1] [0x3fffd9a6c190] +[b08n08:164751] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n08:164751] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n08:164751] [ 4] [b06n06:163678] *** Process received signal *** +[b06n06:163678] Signal: Segmentation fault (11) +[b06n06:163678] Signal code: (3) +[b06n06:163678] Failing at address: 0x14170000000000a0 +[b06n06:163678] [ 0] [0x1000000504d8] +[b06n06:163678] [ 1] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n14:126466] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n14:126466] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n14:126466] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n16:164551] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n16:164551] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n16:164551] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n16:164551] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n05:173081] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n05:173081] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n05:173081] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n05:173081] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n05:173081] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n02:158418] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n02:158418] *** End of error message *** +[a26n07:60172] *** Process received signal *** +[a26n07:60172] Signal: Segmentation fault (11) +[a26n07:60172] Signal code: (3) +[a26n07:60172] Failing at address: 0xb170000000000a0 +[a26n07:60172] [ 0] [0x1000000504d8] +[a26n07:60172] [ 1] [0x3fffd3119d10] +[a26n07:60172] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n07:60172] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n07:60172] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n07:60172] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n07:60172] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n07:60172] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n07:60172] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n07:60172] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n01:123571] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n01:123571] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n01:123571] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n01:123571] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n01:123571] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n03:175064] *** End of error message *** +[0x1000000504d8] +[b12n12:138737] [ 1] [0x3fffcfe8b400] +[b12n12:138737] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n12:138737] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n12:138737] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n12:138737] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n12:138737] [ 6] [a30n17:176414] *** Process received signal *** +[a30n17:176414] Signal: Segmentation fault (11) +[a30n17:176414] Signal code: (3) +[a30n17:176414] Failing at address: 0x14170000000000a0 +[a30n17:176414] [ 0] [0x1000000504d8] +[a30n17:176414] [ 1] [0x3fffc1aec1a0] +[a30n17:176414] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n17:176414] [ 3] [a32n09:155757] *** Process received signal *** +[a32n09:155757] Signal: Segmentation fault (11) +[a32n09:155757] Signal code: (3) +[a32n09:155757] Failing at address: 0xb170000000000a0 +[a32n09:155757] [ 0] [0x1000000504d8] +[a32n09:155757] [ 1] [0x3fffed4484a0] +[a32n09:155757] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n09:155757] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n17:123238] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n17:123238] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n17:123238] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n17:123238] [ 8] [b08n18:164011] *** Process received signal *** +[b08n18:164011] Signal: Segmentation fault (11) +[b08n18:164011] Signal code: (3) +[b08n18:164011] Failing at address: 0xb170000000000a0 +[b08n18:164011] [ 0] [0x1000000504d8] +[b08n18:164011] [ 1] [0x3ffff1a3e3b0] +[b08n18:164011] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n18:164011] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n18:164011] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n18:164011] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n18:164011] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n18:164011] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n18:164011] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n18:164011] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n18:164011] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n18:164011] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n18:164011] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n18:164011] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n18:164011] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n18:164011] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n17:168156] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n17:168156] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n17:168156] [11] [0x3fffc4ebcf30] +[a29n17:169737] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n17:169737] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n17:169737] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n17:169737] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n17:169737] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n06:117525] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n06:117525] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n06:117525] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n07:152085] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n07:152085] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n07:152085] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n07:152085] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n01:165125] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n01:165125] [ 5] [a35n06:138765] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n06:161720] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n06:161720] [ 6] [b11n03:154294] *** Process received signal *** +[b11n03:154294] Signal: Segmentation fault (11) +[b11n03:154294] Signal code: (3) +[b11n03:154294] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n12:141259] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n12:141259] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n12:178596] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n12:178596] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n12:178596] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n12:178596] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n12:178596] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n12:178596] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n12:178596] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n12:178596] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n12:178596] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n12:178596] *** End of error message *** +[b08n14:178327] *** Process received signal *** +[b08n14:178327] Signal: Segmentation fault (11) +[b08n14:178327] Signal code: (3) +[b08n14:178327] Failing at address: 0x14170000000000a0 +[0x3fffe0598d70] +[b23n11:117475] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n11:117475] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n11:117475] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n11:117475] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n18:118682] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n04:158633] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n04:158633] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n04:158633] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n04:158633] [14] [0x3fffe3760230] +[b08n07:168963] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n07:168963] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n07:168963] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n07:168963] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n06:157790] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n06:157790] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n06:157790] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n06:157790] [ 8] [b03n17:176828] *** Process received signal *** +[b03n17:176828] Signal: Segmentation fault (11) +[b03n17:176828] Signal code: (3) +[b03n17:176828] Failing at address: 0xb170000000000a0 +[b03n17:176828] [ 0] [0x1000000504d8] +[b03n17:176828] [ 1] [0x3ffff2af7db0] +[b03n17:176828] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n17:176828] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n17:176828] [ 4] [b21n11:120581] *** Process received signal *** +[b21n11:120581] Signal: Segmentation fault (11) +[b21n11:120581] Signal code: (3) +[b21n11:120581] Failing at address: 0xb170000000000a0 +[b21n11:120581] [ 0] [0x1000000504d8] +[b21n11:120581] [ 1] [0x3ffffc834e80] +[b21n11:120581] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n11:120581] [ 3] [0x3fffc613eb80] +[b23n09:119634] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n09:119634] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n09:119634] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n04:145750] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n04:145750] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n04:145750] [ 8] [0x1000000504d8] +[b02n13:171109] [ 1] [0x3ffff07dce10] +[b02n13:171109] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n13:171109] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n13:171109] [ 4] [b08n10:11489] *** Process received signal *** +[b08n10:11489] Signal: Segmentation fault (11) +[b08n10:11489] Signal code: (3) +[b08n10:11489] Failing at address: 0x14170000000000a0 +[b08n10:11489] [ 0] [0x1000000504d8] +[b08n10:11489] [ 1] [0x3ffffade1bf0] +[b08n10:11489] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n10:11489] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n05:139438] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n05:139438] *** End of error message *** +[a29n07:160802] Signal: Segmentation fault (11) +[a29n07:160802] Signal code: (3) +[a29n07:160802] Failing at address: 0x14170000000000a0 +[a29n07:160802] [ 0] [0x1000000504d8] +[a29n07:160802] [ 1] [0x3ffff14114e0] +[a29n07:160802] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n07:160802] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n07:160802] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n03:46687] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n03:46687] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n03:46687] [ 6] [b05n17:177403] [ 0] [0x1000000504d8] +[b05n17:177403] [ 1] [0x3fffd22b7bb0] +[b05n17:177403] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n17:177403] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n13:153678] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n13:153678] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n13:153678] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n13:153678] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n13:153678] [ 8] [a35n16:148188] *** Process received signal *** +[a35n16:148188] Signal: Segmentation fault (11) +[a35n16:148188] Signal code: (3) +[a35n16:148188] Failing at address: 0x14170000000000a0 +[a35n16:148188] [ 0] [0x1000000504d8] +[a35n16:148188] [ 1] [0x3fffd28f2290] +[a35n16:148188] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n16:148188] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n11:03987] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n11:03987] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n11:03987] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n11:03987] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n11:03987] [13] [0x3fffc53702c0] +[b22n03:120670] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n03:120670] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n03:120670] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n03:120670] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n03:120670] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n03:120670] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n03:120670] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n03:120670] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n03:120670] [10] [0x3fffd1b83a20] +[b16n17:119977] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n17:119977] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n17:119977] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n17:119977] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n16:08386] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n16:08386] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n16:08386] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n16:08386] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n18:165696] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n18:165696] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n18:165696] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n18:165696] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n18:165696] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n18:165696] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n18:165696] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n18:165696] [11] [b13n11:134522] *** Process received signal *** +[b13n11:134522] Signal: Segmentation fault (11) +[b13n11:134522] Signal code: (3) +[b13n11:134522] Failing at address: 0x14170000000000a0 +[b13n11:134522] [ 0] [0x1000000504d8] +[b13n11:134522] [ 1] [0x3fffc353b040] +[b13n11:134522] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n11:134522] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n11:134522] [ 4] [b22n04:44755] [ 0] [0x1000000504d8] +[b22n04:44755] [ 1] [0x3fffdadccdc0] +[b22n04:44755] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n04:44755] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n04:44755] [ 4] [a28n14:08398] *** Process received signal *** +[a28n14:08398] Signal: Segmentation fault (11) +[a28n14:08398] Signal code: (3) +[a28n14:08398] Failing at address: 0xb170000000000a0 +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n05:139910] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n05:139910] [ 4] [a28n18:179171] *** Process received signal *** +[a28n18:179171] Signal: Segmentation fault (11) +[a28n18:179171] Signal code: (3) +[a28n18:179171] Failing at address: 0x14170000000000a0 +[a28n18:179171] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n12:123558] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n12:123558] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n12:123558] [ 6] [0x3ffff73ae0d0] +[b18n13:123055] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n13:123055] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n13:123055] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n13:123055] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n10:140412] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n10:140412] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n10:140412] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n08:159022] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n08:159022] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n08:159022] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n11:130806] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n11:130806] [ 5] [b20n18:123412] *** Process received signal *** +[b20n18:123412] Signal: Segmentation fault (11) +[b20n18:123412] Signal code: (3) +[b20n18:123412] Failing at address: 0xb170000000000a0 +[b20n18:123412] [ 0] [0x1000000504d8] +[b20n18:123412] [ 1] [0x3fffd6dad1f0] +[b20n18:123412] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n18:123412] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n13:164692] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n13:164692] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n13:164692] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n13:164692] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n13:164692] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n13:164692] [12] [0x3fffdf97ccd0] +[a28n05:04520] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n05:04520] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n05:04520] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n16:154184] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n16:154184] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n16:154184] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n12:145499] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n12:145499] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n12:145499] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n10:131274] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n10:131274] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n10:131274] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n10:131274] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n10:131274] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n10:131274] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n07:140136] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n07:140136] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n07:140136] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n07:140136] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n07:140136] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n07:140136] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n07:140136] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n07:140136] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n07:140136] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n07:140136] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n07:140136] [14] [b18n15:120132] [ 0] [0x1000000504d8] +[b18n15:120132] [ 1] [0x3fffce9f5ac0] +[b18n15:120132] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n15:120132] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n15:120132] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n11:156153] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n11:156153] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n11:156153] [13] [0x1000000504d8] +[b23n05:120231] [ 1] [0x3fffc122ebe0] +[b23n05:120231] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n05:120231] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n05:120231] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n05:120231] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n07:131674] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n07:131674] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n07:131674] *** End of error message *** +[a30n18:10720] *** Process received signal *** +[a30n18:10720] Signal: Segmentation fault (11) +[a30n18:10720] Signal code: (3) +[a30n18:10720] Failing at address: 0xb170000000000a0 +[a30n18:10720] [ 0] [0x1000000504d8] +[a30n18:10720] [ 1] [a33n18:149117] *** Process received signal *** +[a33n18:149117] Signal: Segmentation fault (11) +[a33n18:149117] Signal code: (3) +[a33n18:149117] Failing at address: 0x14170000000000a0 +[a33n18:149117] [ 0] [0x1000000504d8] +[a33n18:149117] [ 1] [0x3ffffceae670] +[a33n18:149117] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n18:149117] [ 3] [b17n10:126613] *** Process received signal *** +[b17n10:126613] Signal: Segmentation fault (11) +[b17n10:126613] Signal code: (3) +[b17n10:126613] Failing at address: 0xb170000000000a0 +[b17n10:126613] [ 0] [0x1000000504d8] +[b17n10:126613] [ 1] [0x3fffe2834c70] +[b17n10:126613] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n10:126613] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n10:126613] [ 4] [b13n13:133624] *** Process received signal *** +[b13n13:133624] Signal: Segmentation fault (11) +[b13n13:133624] Signal code: (3) +[b13n13:133624] Failing at address: 0x14170000000000a0 +[b13n13:133624] [ 0] [b02n10:01250] *** Process received signal *** +[b02n10:01250] Signal: Segmentation fault (11) +[b02n10:01250] Signal code: (3) +[b02n10:01250] Failing at address: 0x14170000000000a0 +[b02n10:01250] [ 0] [0x1000000504d8] +[b02n10:01250] [ 1] [0x3fffe703e840] +[b02n10:01250] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n10:01250] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n10:01250] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n10:01250] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n10:01250] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n06:119116] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n06:119116] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n06:119116] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n06:119116] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n10:135855] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n10:135855] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n10:135855] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n10:135855] [11] [a26n14:51450] *** Process received signal *** +[a26n14:51450] Signal: Segmentation fault (11) +[a26n14:51450] Signal code: (3) +[a26n14:51450] Failing at address: 0x14170000000000a0 +[a26n14:51450] [ 0] [0x1000000504d8] +[a26n14:51450] [ 1] [a34n07:165091] *** Process received signal *** +[a34n07:165091] Signal: Segmentation fault (11) +[a34n07:165091] Signal code: (3) +[a34n07:165091] Failing at address: 0x14170000000000a0 +[a34n07:165091] [ 0] [0x1000000504d8] +[a34n07:165091] [ 1] [0x3fffde4cd9c0] +[a34n07:165091] [ 2] [b05n10:10922] *** Process received signal *** +[b05n10:10922] Signal: Segmentation fault (11) +[b05n10:10922] Signal code: (3) +[b05n10:10922] Failing at address: 0xb170000000000a0 +[b05n10:10922] [ 0] [0x1000000504d8] +[b05n10:10922] [ 1] [0x3fffe0a3e330] +[b05n10:10922] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n10:120979] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n10:120979] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n10:120979] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n10:120979] [ 8] [b04n03:09641] *** Process received signal *** +[b04n03:09641] Signal: Segmentation fault (11) +[b04n03:09641] Signal code: (3) +[b04n03:09641] Failing at address: 0xb170000000000a0 +[b04n03:09641] [ 0] [0x1000000504d8] +[b04n03:09641] [ 1] [0x3fffff36cde0] +[b04n03:09641] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n03:09641] [ 3] [b05n05:173318] *** Process received signal *** +[b05n05:173318] Signal: Segmentation fault (11) +[b05n05:173318] Signal code: (3) +[b05n05:173318] Failing at address: 0xb170000000000a0 +[b05n05:173318] [ 0] [0x1000000504d8] +[b05n05:173318] [ 1] [0x3fffccb59d40] +[b05n05:173318] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n05:173318] [ 3] [a33n13:168162] *** Process received signal *** +[a33n13:168162] Signal: Segmentation fault (11) +[a33n13:168162] Signal code: (3) +[a33n13:168162] Failing at address: 0x14170000000000a0 +[a33n13:168162] [ 0] [0x1000000504d8] +[a33n13:168162] [ 1] [a35n05:149466] *** Process received signal *** +[a35n05:149466] Signal: Segmentation fault (11) +[a35n05:149466] Signal code: (3) +[a35n05:149466] Failing at address: 0x14170000000000a0 +[a35n05:149466] [ 0] [0x1000000504d8] +[a35n05:149466] [ 1] [0x3fffda9db330] +[a35n05:149466] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n16:124959] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n16:124959] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n16:124959] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n16:124959] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n12:160188] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n12:160188] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n12:160188] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n12:160188] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n12:160188] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n12:160188] *** End of error message *** +[ 1] [0x3fffdd993000] +[b09n09:170658] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n09:170658] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n10:122761] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n10:122761] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n10:122761] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n10:122761] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n11:123522] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n11:123522] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n11:123522] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n11:123522] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n11:123522] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n11:123522] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n11:123522] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n11:123522] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n11:123522] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n11:123522] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n14:126120] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n14:126120] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n14:126120] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n02:139630] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n02:139630] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n02:139630] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n02:139630] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n02:139630] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n02:139630] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n02:139630] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n02:139630] [13] [b09n17:163178] *** Process received signal *** +[b09n17:163178] Signal: Segmentation fault (11) +[b09n17:163178] Signal code: (3) +[b09n17:163178] Failing at address: 0xb170000000000a0 +[b09n17:163178] [ 0] [0x1000000504d8] +[b09n17:163178] [ 1] [0x3fffefff5bf0] +[b09n17:163178] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n17:163178] [ 3] [b22n07:120697] *** Process received signal *** +[b22n07:120697] Signal: Segmentation fault (11) +[b22n07:120697] Signal code: (3) +[b22n07:120697] Failing at address: 0xb170000000000a0 +[b22n07:120697] [ 0] [0x1000000504d8] +[b22n07:120697] [ 1] [0x3fffc8b7f580] +[b22n07:120697] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n07:120697] [ 3] [b04n07:02841] *** Process received signal *** +[b04n07:02841] Signal: Segmentation fault (11) +[b04n07:02841] Signal code: (3) +[b04n07:02841] Failing at address: 0x5170000000000a0 +[b04n07:02841] [ 0] [0x1000000504d8] +[b04n07:02841] [ 1] [0x3fffd0fdcee0] +[b04n07:02841] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n07:02841] [ 3] [b17n03:135036] *** Process received signal *** +[b17n03:135036] Signal: Segmentation fault (11) +[b17n03:135036] Signal code: (3) +[b17n03:135036] Failing at address: 0xb170000000000a0 +[b17n03:135036] [ 0] [b17n09:128494] *** Process received signal *** +[b17n09:128494] Signal: Segmentation fault (11) +[b17n09:128494] Signal code: (3) +[b17n09:128494] Failing at address: 0x14170000000000a0 +[b17n09:128494] [ 0] [0x1000000504d8] +[b17n09:128494] [ 1] [0x3fffe33b6880] +[b17n09:128494] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n09:128494] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n09:128494] [ 4] [b15n08:118554] *** Process received signal *** +[b15n08:118554] Signal: Segmentation fault (11) +[b15n08:118554] Signal code: (3) +[b15n08:118554] Failing at address: 0xb170000000000a0 +[b15n08:118554] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n02:166745] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n02:166745] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n02:166745] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n02:166745] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n02:166745] *** End of error message *** +[a33n03:147536] *** Process received signal *** +[a33n03:147536] Signal: Segmentation fault (11) +[a33n03:147536] Signal code: (3) +[a33n03:147536] Failing at address: 0x14170000000000a0 +[a32n08:165281] *** Process received signal *** +[a32n08:165281] Signal: Segmentation fault (11) +[a32n08:165281] Signal code: (3) +[a32n08:165281] Failing at address: 0xb170000000000a0 +[a32n08:165281] [ 0] [0x1000000504d8] +[a32n08:165281] [ 1] [0x3ffffd561f00] +[a32n08:165281] [ 2] [a35n12:145263] *** Process received signal *** +[b15n11:134749] *** Process received signal *** +[b15n11:134749] Signal: Segmentation fault (11) +[b15n11:134749] Signal code: (3) +[b15n11:134749] Failing at address: 0xb170000000000a0 +[b15n11:134749] [ 0] [0x1000000504d8] +[b15n11:134749] [ 1] [0x3fffede77140] +[b15n11:134749] [ 2] [b12n18:123867] *** Process received signal *** +[b12n18:123867] Signal: Segmentation fault (11) +[b12n18:123867] Signal code: (3) +[b12n18:123867] Failing at address: 0x14170000000000a0 +[b12n18:123867] [ 0] [0x1000000504d8] +[b12n18:123867] [ 1] [b06n10:176261] *** Process received signal *** +[b06n10:176261] Signal: Segmentation fault (11) +[b06n10:176261] Signal code: (3) +[b06n10:176261] Failing at address: 0xb170000000000a0 +[b06n10:176261] [ 0] [0x1000000504d8] +[b06n10:176261] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n08:124116] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n08:124116] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n08:124116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n08:124116] [ 7] [b14n14:122443] *** Process received signal *** +[b14n14:122443] Signal: Segmentation fault (11) +[b14n14:122443] Signal code: (3) +[b14n14:122443] Failing at address: 0xb170000000000a0 +[b14n14:122443] [ 0] [0x1000000504d8] +[b14n14:122443] [ 1] [b18n12:120401] *** Process received signal *** +[b18n12:120401] Signal: Segmentation fault (11) +[b18n12:120401] Signal code: (3) +[b18n12:120401] Failing at address: 0xb170000000000a0 +[b18n12:120401] [ 0] [0x1000000504d8] +[b18n12:120401] [ 1] [0x3fffd2edc5c0] +[b18n12:120401] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n12:120401] [ 3] [b09n05:173822] *** Process received signal *** +[b09n05:173822] Signal: Segmentation fault (11) +[b09n05:173822] Signal code: (3) +[b09n05:173822] Failing at address: 0xb170000000000a0 +[b09n05:173822] [ 0] [0x1000000504d8] +[b09n05:173822] [ 1] [b12n16:129237] *** Process received signal *** +[b12n16:129237] Signal: Segmentation fault (11) +[b12n16:129237] Signal code: (3) +[b12n16:129237] Failing at address: 0xb170000000000a0 +[b12n16:129237] [ 0] [0x1000000504d8] +[b12n16:129237] [ 1] [0x3fffc8efd9e0] +[b12n16:129237] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n16:129237] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n16:129237] [ 4] [b17n04:136610] *** Process received signal *** +[b17n04:136610] Signal: Segmentation fault (11) +[b17n04:136610] Signal code: (3) +[b17n04:136610] Failing at address: 0xb170000000000a0 +[b17n04:136610] [ 0] [0x1000000504d8] +[b17n04:136610] [ 1] [0x3ffff49b6c50] +[b17n04:136610] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n04:136610] [ 3] [a29n13:165387] *** Process received signal *** +[a29n13:165387] Signal: Segmentation fault (11) +[a29n13:165387] Signal code: (3) +[a29n13:165387] Failing at address: 0xb170000000000a0 +[a29n13:165387] [ 0] [b20n03:124680] *** Process received signal *** +[b20n03:124680] Signal: Segmentation fault (11) +[b20n03:124680] Signal code: (3) +[b20n03:124680] Failing at address: 0xb170000000000a0 +[b20n03:124680] [ 0] [0x1000000504d8] +[b20n03:124680] [ 1] [a34n06:148310] *** Process received signal *** +[a34n06:148310] Signal: Segmentation fault (11) +[a34n06:148310] Signal code: (3) +[a34n06:148310] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n07:116115] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n07:116115] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n07:116115] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n07:116115] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n07:116115] [ 8] [a33n12:168645] *** Process received signal *** +[a33n12:168645] Signal: Segmentation fault (11) +[a33n12:168645] Signal code: (3) +[a33n12:168645] Failing at address: 0xb170000000000a0 +[a33n12:168645] [ 0] [0x1000000504d8] +[a33n12:168645] [ 1] [0x3fffe4248600] +[a33n12:168645] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n12:168645] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n11:23065] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n11:23065] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n14:116415] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n05:53375] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n05:53375] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n05:53375] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n05:53375] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n05:53375] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n07:156144] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n07:156144] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n18:121304] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n18:121304] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n04:172231] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n04:172231] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n17:08308] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n17:08308] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n17:08308] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n17:08308] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n17:08308] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n17:08308] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n17:08308] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n17:08308] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n17:08308] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n16:147288] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n16:147288] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n12:177495] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n12:177495] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n12:177495] [11] [b09n04:177599] *** Process received signal *** +[b09n04:177599] Signal: Segmentation fault (11) +[b09n04:177599] Signal code: (3) +[b09n04:177599] Failing at address: 0x14170000000000a0 +[b09n04:177599] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n05:05609] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n05:05609] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n05:05609] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n05:05609] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n05:05609] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n05:05609] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n05:05609] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n05:05609] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n05:05609] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n05:05609] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n05:05609] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n09:62480] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n09:62480] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n09:62480] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n09:62480] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n09:62480] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n09:123417] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n09:123417] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n09:143901] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n09:143901] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n09:143901] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n14:139846] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n14:139846] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n14:139846] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n04:117355] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n04:117355] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n04:117355] [ 5] [a33n06:165305] *** Process received signal *** +[a33n06:165305] Signal: Segmentation fault (11) +[a33n06:165305] Signal code: (3) +[a33n06:165305] Failing at address: 0x14170000000000a0 +[a33n06:165305] [ 0] [0x1000000504d8] +[a33n06:165305] [ 1] [0x3ffffc3f9f80] +[a33n06:165305] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n06:165305] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n06:165305] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n06:165305] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n06:165305] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n06:165305] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n06:165305] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n06:165305] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n06:165305] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n06:165305] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n06:165305] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n06:165305] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n06:165305] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n06:165305] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n16:168049] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n16:168049] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n16:168049] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n16:168049] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n16:168049] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n13:131758] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n13:131758] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n13:131758] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n13:131758] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n13:131758] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n18:03314] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n18:03314] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n18:03314] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n18:03314] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n18:03314] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n18:03314] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n18:03314] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n18:03314] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n18:03314] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n18:03314] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n18:03314] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n13:130060] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n13:130060] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n13:130060] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n13:130060] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n13:130060] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n13:130060] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n13:130060] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n13:130060] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n13:130060] *** End of error message *** +[a33n07:149479] *** Process received signal *** +[a33n07:149479] Signal: Segmentation fault (11) +[a33n07:149479] Signal code: (3) +[a33n07:149479] Failing at address: 0x14170000000000a0 +[a33n07:149479] [ 0] [0x1000000504d8] +[a33n07:149479] [ 1] [0x3fffffdabca0] +[a33n07:149479] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n07:149479] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n07:149479] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n07:149479] [ 5] [b20n12:119806] *** Process received signal *** +[b20n12:119806] Signal: Segmentation fault (11) +[b20n12:119806] Signal code: (3) +[b20n12:119806] Failing at address: 0xb170000000000a0 +[b20n12:119806] [ 0] [0x1000000504d8] +[b20n12:119806] [ 1] [0x3fffe1330630] +[b20n12:119806] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n12:119806] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n13:177640] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n13:177640] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n13:177640] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n13:177640] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n13:177640] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n14:120639] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n14:120639] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n08:144853] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n08:144853] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n08:144853] [ 9] [b05n13:10198] *** Process received signal *** +[b05n13:10198] Signal: Segmentation fault (11) +[b05n13:10198] Signal code: (3) +[b05n13:10198] Failing at address: 0x14170000000000a0 +[b05n13:10198] [ 0] [0x1000000504d8] +[b05n13:10198] [ 1] [0x3ffff4519560] +[b05n13:10198] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n13:122187] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n13:122187] [10] [b10n10:126193] *** Process received signal *** +[b10n10:126193] Signal: Segmentation fault (11) +[b10n10:126193] Signal code: (3) +[b10n10:126193] Failing at address: 0xb170000000000a0 +[b10n10:126193] [ 0] [0x1000000504d8] +[b10n10:126193] [ 1] [0x3fffc294d060] +[b10n10:126193] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n10:126193] [ 3] [b19n16:130375] *** Process received signal *** +[b19n16:130375] Signal: Segmentation fault (11) +[b19n16:130375] Signal code: (3) +[b19n16:130375] Failing at address: 0xb170000000000a0 +[b19n16:130375] [ 0] [0x1000000504d8] +[b19n16:130375] [ 1] [0x3fffc3d2e9b0] +[b19n16:130375] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n16:130375] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n15:07536] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n15:07536] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n15:07536] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n15:07536] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n15:07536] *** End of error message *** +[a35n08:138174] *** Process received signal *** +[a35n08:138174] Signal: Segmentation fault (11) +[a35n08:138174] Signal code: (3) +[a35n08:138174] Failing at address: 0xb170000000000a0 +[a35n08:138174] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n09:124192] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n09:124192] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n17:125808] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n17:125808] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n17:125808] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n07:174561] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n07:174561] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n12:180117] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n12:180117] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n12:180117] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n13:138089] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n15:32401] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n15:32401] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n07:173042] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n07:173042] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n07:173042] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n07:173042] [13] [b22n15:120058] *** Process received signal *** +[b22n15:120058] Signal: Segmentation fault (11) +[b22n15:120058] Signal code: (3) +[b22n15:120058] Failing at address: 0x14170000000000a0 +[b22n15:120058] [ 0] [0x1000000504d8] +[b22n15:120058] [ 1] [0x3fffce0a5ee0] +[b22n15:120058] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n15:120058] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n15:120058] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n15:120058] [ 5] [b14n10:124923] *** Process received signal *** +[b14n10:124923] Signal: Segmentation fault (11) +[b14n10:124923] Signal code: (3) +[b14n10:124923] Failing at address: 0x14170000000000a0 +[b14n10:124923] [ 0] [0x1000000504d8] +[b14n10:124923] [ 1] [0x3fffda6a9270] +[b14n10:124923] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n10:124923] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n10:124923] [ 4] [b17n11:124445] *** Process received signal *** +[b17n11:124445] Signal: Segmentation fault (11) +[b17n11:124445] Signal code: (3) +[b17n11:124445] Failing at address: 0x14170000000000a0 +[b17n11:124445] [ 0] [0x1000000504d8] +[b17n11:124445] [ 1] [0x3fffdfa171a0] +[b17n11:124445] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n11:124445] [ 3] [a33n14:136137] *** Process received signal *** +[a33n14:136137] Signal: Segmentation fault (11) +[a33n14:136137] Signal code: (3) +[a33n14:136137] Failing at address: 0x14170000000000a0 +[a33n14:136137] [ 0] [0x1000000504d8] +[a33n14:136137] [ 1] [0x3fffff34d280] +[a33n14:136137] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n14:136137] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n14:136137] [ 4] [b05n01:178523] *** Process received signal *** +[b05n01:178523] Signal: Segmentation fault (11) +[b05n01:178523] Signal code: (3) +[b05n01:178523] Failing at address: 0x14170000000000a0 +[b05n01:178523] [ 0] [0x1000000504d8] +[b05n01:178523] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n12:164384] *** End of error message *** +[b09n01:163026] *** Process received signal *** +[b09n01:163026] Signal: Segmentation fault (11) +[b09n01:163026] Signal code: (3) +[b09n01:163026] Failing at address: 0xb170000000000a0 +[b09n01:163026] [ 0] [0x1000000504d8] +[b09n01:163026] [ 1] [0x3ffff1e72650] +[b09n01:163026] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n01:163026] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n01:163026] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n02:167597] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n02:167597] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n02:167597] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n02:167597] [14] [b04n18:176808] *** Process received signal *** +[b04n18:176808] Signal: Segmentation fault (11) +[b04n18:176808] Signal code: (3) +[b04n18:176808] Failing at address: 0xb170000000000a0 +[b04n18:176808] [ 0] [0x1000000504d8] +[b04n18:176808] [ 1] [b06n13:166925] *** Process received signal *** +[b06n13:166925] Signal: Segmentation fault (11) +[b06n13:166925] Signal code: (3) +[b06n13:166925] Failing at address: 0x14170000000000a0 +[b06n13:166925] [ 0] [0x1000000504d8] +[b06n13:166925] [ 1] [0x3ffff79b25f0] +[b06n13:166925] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n13:166925] [ 3] [b11n06:164013] *** Process received signal *** +[b11n06:164013] Signal: Segmentation fault (11) +[b11n06:164013] Signal code: (3) +[b11n06:164013] Failing at address: 0x14170000000000a0 +[b11n06:164013] [ 0] [0x1000000504d8] +[b11n06:164013] [ 1] [0x3fffc8e484b0] +[b11n06:164013] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n06:164013] [ 3] [a35n15:148167] *** Process received signal *** +[a35n15:148167] Signal: Segmentation fault (11) +[a35n15:148167] Signal code: (3) +[a35n15:148167] Failing at address: 0x5170000000000a0 +[a35n15:148167] [ 0] [0x1000000504d8] +[a35n15:148167] [ 1] [0x3ffffc82bb10] +[a35n15:148167] [ 2] [b05n07:08398] *** Process received signal *** +[b05n07:08398] Signal: Segmentation fault (11) +[b05n07:08398] Signal code: (3) +[b05n07:08398] Failing at address: 0xb170000000000a0 +[b05n07:08398] [ 0] [0x1000000504d8] +[b05n07:08398] [ 1] [0x3fffc4677560] +[b05n07:08398] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n07:08398] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n07:08398] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n07:08398] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n07:08398] [ 6] [a36n15:151252] *** Process received signal *** +[a36n15:151252] Signal: Segmentation fault (11) +[a36n15:151252] Signal code: (3) +[a36n15:151252] Failing at address: 0x14170000000000a0 +[a36n15:151252] [ 0] [0x1000000504d8] +[a36n15:151252] [ 1] [0x3ffffab7b940] +[a36n15:151252] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n15:151252] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n18:173979] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n18:173979] [ 5] [b04n02:177548] *** Process received signal *** +[b04n02:177548] Signal: Segmentation fault (11) +[b04n02:177548] Signal code: (3) +[b04n02:177548] Failing at address: 0xb170000000000a0 +[b04n02:177548] [ 0] [0x1000000504d8] +[b04n02:177548] [ 1] [0x3fffe28456e0] +[b04n02:177548] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n02:177548] [ 3] [a34n14:150133] *** Process received signal *** +[a34n14:150133] Signal: Segmentation fault (11) +[a34n14:150133] Signal code: (3) +[a34n14:150133] Failing at address: 0xb170000000000a0 +[a34n14:150133] [ 0] [0x1000000504d8] +[a34n14:150133] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n15:119252] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n15:119252] [10] [a29n11:162472] *** Process received signal *** +[a29n11:162472] Signal: Segmentation fault (11) +[a29n11:162472] Signal code: (3) +[a29n11:162472] Failing at address: 0x14170000000000a0 +[a29n11:162472] [ 0] [0x1000000504d8] +[a29n11:162472] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n14:119483] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n14:119483] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n14:119483] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n14:119483] [14] [b23n10:126789] *** Process received signal *** +[b23n10:126789] Signal: Segmentation fault (11) +[b23n10:126789] Signal code: (3) +[b23n10:126789] Failing at address: 0xb170000000000a0 +[b23n10:126789] [ 0] [0x1000000504d8] +[b23n10:126789] [ 1] [0x3fffcb18bd10] +[b23n10:126789] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n10:126789] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n10:126789] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n10:126789] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n10:126789] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n10:126789] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n10:126789] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n10:126789] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n10:126789] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n10:126789] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n10:126789] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n10:126789] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n10:126789] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n10:126789] *** End of error message *** +[b09n18:164797] *** Process received signal *** +[b09n18:164797] Signal: Segmentation fault (11) +[b09n18:164797] Signal code: (3) +[b09n18:164797] Failing at address: 0xb170000000000a0 +[b09n18:164797] [ 0] [0x1000000504d8] +[b09n18:164797] [ 1] [0x3fffecd4ae40] +[b09n18:164797] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n18:164797] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n13:117510] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n13:117510] [ 6] [b22n02:119533] *** Process received signal *** +[b22n02:119533] Signal: Segmentation fault (11) +[b22n02:119533] Signal code: (3) +[b22n02:119533] Failing at address: 0x14170000000000a0 +[b22n02:119533] [ 0] [0x1000000504d8] +[b22n02:119533] [ 1] [a28n17:168159] *** Process received signal *** +[a28n17:168159] Signal: Segmentation fault (11) +[a28n17:168159] Signal code: (3) +[a28n17:168159] Failing at address: 0xb170000000000a0 +[a28n17:168159] [ 0] [b17n05:125614] *** Process received signal *** +[b17n05:125614] Signal: Segmentation fault (11) +[b17n05:125614] Signal code: (3) +[b17n05:125614] Failing at address: 0xb170000000000a0 +[b17n05:125614] [ 0] [0x1000000504d8] +[b17n05:125614] [ 1] [0x3fffcc617fa0] +[b17n05:125614] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n05:125614] [ 3] [ 0] [0x1000000504d8] +[b20n04:126349] [ 1] [0x3fffefc0c950] +[b20n04:126349] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n04:126349] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n04:126349] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n04:126349] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n04:126349] [ 6] [b09n12:178718] *** Process received signal *** +[b09n12:178718] Signal: Segmentation fault (11) +[b09n12:178718] Signal code: (3) +[b09n12:178718] Failing at address: 0x14170000000000a0 +[b09n12:178718] [ 0] [0x1000000504d8] +[b09n12:178718] [ 1] [0x3fffed725d90] +[b09n12:178718] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n12:178718] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n16:173549] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n16:173549] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n16:173549] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n18:08429] *** End of error message *** +[b13n10:134732] *** Process received signal *** +[b13n10:134732] Signal: Segmentation fault (11) +[b13n10:134732] Signal code: (3) +[b13n10:134732] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n11:163262] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n04:153965] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n04:153965] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n04:153965] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n06:135851] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n06:135851] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n06:135851] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n06:135851] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n11:153420] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n11:153420] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n11:153420] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n03:158660] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n03:158660] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n03:158660] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n18:177223] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n18:177223] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n18:177223] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n18:177223] [10] [b01n10:173682] *** Process received signal *** +[b01n10:173682] Signal: Segmentation fault (11) +[b01n10:173682] Signal code: (3) +[b01n10:173682] Failing at address: 0x14170000000000a0 +[b01n10:173682] [ 0] [0x1000000504d8] +[b01n10:173682] [ 1] [0x3fffe5fea140] +[b01n10:173682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n10:173682] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n16:149064] *** End of error message *** +[b13n17:129932] *** Process received signal *** +[b13n17:129932] Signal: Segmentation fault (11) +[b13n17:129932] Signal code: (3) +[b13n17:129932] Failing at address: 0xb170000000000a0 +[b13n17:129932] [ 0] [0x1000000504d8] +[b13n17:129932] [ 1] [b04n04:179173] *** Process received signal *** +[b04n04:179173] Signal: Segmentation fault (11) +[b04n04:179173] Signal code: (3) +[b04n04:179173] Failing at address: 0x14170000000000a0 +[b04n04:179173] [ 0] [0x1000000504d8] +[b04n04:179173] [ 1] [0x3fffeeada230] +[b04n04:179173] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n04:179173] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n12:120130] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n12:120130] [ 6] [b19n13:123888] *** Process received signal *** +[b19n13:123888] Signal: Segmentation fault (11) +[b19n13:123888] Signal code: (3) +[b19n13:123888] Failing at address: 0x14170000000000a0 +[b19n13:123888] [ 0] [0x1000000504d8] +[b19n13:123888] [ 1] [0x3fffdc7849f0] +[b19n13:123888] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n13:123888] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n09:127999] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n09:127999] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n09:127999] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n18:138475] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n18:138475] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n18:138475] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n17:171200] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n17:171200] *** End of error message *** +[0x1000000504d8] +[b21n08:116172] [ 1] [0x3fffdd397f90] +[b21n08:116172] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n08:116172] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n08:116172] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n08:116172] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n08:116172] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n16:165922] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n16:165922] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n16:165922] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n16:165922] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n07:133377] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n04:52705] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n04:52705] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n05:54536] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n05:54536] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n05:54536] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n18:07425] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n18:07425] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n18:07425] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n18:07425] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n18:07425] *** End of error message *** +[ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n16:127353] *** Process received signal *** +[b17n16:127353] Signal: Segmentation fault (11) +[b17n16:127353] Signal code: (3) +[b17n16:127353] Failing at address: 0xb170000000000a0 +[b17n16:127353] [ 0] [0x1000000504d8] +[b17n16:127353] [ 1] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n11:124435] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n11:124435] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n11:124435] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n11:124435] [ 6] [0x1000000504d8] +[b11n04:166206] [ 1] [0x3fffef0427e0] +[b11n04:166206] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n04:166206] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n04:166206] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n04:166206] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n17:124167] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n17:124167] [ 5] [b22n13:122032] *** Process received signal *** +[b22n13:122032] Signal: Segmentation fault (11) +[b22n13:122032] Signal code: (3) +[b22n13:122032] Failing at address: 0x14170000000000a0 +[b22n13:122032] [ 0] [0x1000000504d8] +[b22n13:122032] [ 1] [0x3fffcacf4780] +[b22n13:122032] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n13:122032] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n16:128443] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n16:128443] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n16:128443] [ 9] [a31n06:167368] *** Process received signal *** +[a31n06:167368] Signal: Segmentation fault (11) +[a31n06:167368] Signal code: (3) +[a31n06:167368] Failing at address: 0x14170000000000a0 +[a31n06:167368] [ 0] [0x3fffe331b370] +[a26n18:69155] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n18:69155] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n18:69155] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n18:69155] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n10:124502] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n10:124502] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n10:124502] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n10:124502] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n10:124502] [14] [b15n12:132100] *** Process received signal *** +[b15n12:132100] Signal: Segmentation fault (11) +[b15n12:132100] Signal code: (3) +[b15n12:132100] Failing at address: 0xb170000000000a0 +[b15n12:132100] [ 0] [b09n10:169683] *** Process received signal *** +[b09n10:169683] Signal: Segmentation fault (11) +[b09n10:169683] Signal code: (3) +[b09n10:169683] Failing at address: 0xb170000000000a0 +[b09n10:169683] [ 0] [0x1000000504d8] +[b09n10:169683] [ 1] [0x3ffff389cfd0] +[b09n10:169683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n10:169683] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n10:169683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n10:169683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n10:169683] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n01:121747] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n03:38501] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n03:38501] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n03:38501] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n03:38501] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n03:38501] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n14:08951] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n14:08951] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n14:08951] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n14:08951] [14] [b23n13:114398] *** Process received signal *** +[b23n13:114398] Signal: Segmentation fault (11) +[b23n13:114398] Signal code: (3) +[b23n13:114398] Failing at address: 0x14170000000000a0 +[b23n13:114398] [ 0] [0x1000000504d8] +[b23n13:114398] [ 1] [0x3fffd2247790] +[b23n13:114398] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n13:114398] [ 3] [0x1000000504d8] +[a27n01:48805] [ 1] [0x3fffe5a499e0] +[a27n01:48805] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n01:48805] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n01:48805] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n12:12761] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n12:12761] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n12:167678] [a29n15:177058] *** Process received signal *** +[a29n15:177058] Signal: Segmentation fault (11) +[a29n15:177058] Signal code: (3) +[a29n15:177058] Failing at address: 0xb170000000000a0 +[a29n15:177058] [ 0] [0x1000000504d8] +[a29n15:177058] [ 1] [b01n17:179116] *** Process received signal *** +[b01n17:179116] Signal: Segmentation fault (11) +[b01n17:179116] Signal code: (3) +[b01n17:179116] Failing at address: 0x14170000000000a0 +[b01n17:179116] [ 0] [0x1000000504d8] +[b01n17:179116] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n11:160351] *** End of error message *** +[b19n12:122872] *** Process received signal *** +[b19n12:122872] Signal: Segmentation fault (11) +[b19n12:122872] Signal code: (3) +[b19n12:122872] Failing at address: 0xb170000000000a0 +[b19n12:122872] [ 0] [0x1000000504d8] +[b19n12:122872] [ 1] [0x3ffffc4a8990] +[b19n12:122872] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n12:122872] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n12:122872] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n10:142929] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n10:142929] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n10:142929] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n17:149904] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n17:149904] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n17:149904] [ 8] [a29n16:157889] [ 0] [0x1000000504d8] +[a29n16:157889] [ 1] [0x3ffff2ccd070] +[a29n16:157889] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n16:157889] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n16:157889] [ 4] [0x3ffffa22fd40] +[b21n16:116526] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n16:116526] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n16:116526] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n16:116526] [ 5] [b13n05:135165] *** Process received signal *** +[b13n05:135165] Signal: Segmentation fault (11) +[b13n05:135165] Signal code: (3) +[b13n05:135165] Failing at address: 0xb170000000000a0 +[b13n05:135165] [ 0] [0x1000000504d8] +[b13n05:135165] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n07:118167] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n07:118167] [ 5] [b18n14:128871] *** Process received signal *** +[b18n14:128871] Signal: Segmentation fault (11) +[b18n14:128871] Signal code: (3) +[b18n14:128871] Failing at address: 0x14170000000000a0 +[b18n14:128871] [ 0] [0x1000000504d8] +[b18n14:128871] [ 1] [0x3fffc8eda2a0] +[b18n14:128871] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n14:128871] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n02:06561] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n02:06561] [ 4] [b19n04:130749] *** Process received signal *** +[b19n04:130749] Signal: Segmentation fault (11) +[b19n04:130749] Signal code: (3) +[b19n04:130749] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n11:62421] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n11:62421] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n11:62421] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n11:62421] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n11:62421] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n09:03928] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n09:03928] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n09:03928] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n09:03928] [14] [0x3ffffd02bde0] +[b16n14:133632] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n14:133632] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n14:133632] [ 4] [0x1000000504d8] +[b01n18:179699] [ 1] [0x3fffe3b7f280] +[b01n18:179699] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n18:179699] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n18:179699] [ 4] [ 0] [0x1000000504d8] +[b02n14:169522] [ 1] [0x3fffdc71fe10] +[b02n14:169522] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n14:169522] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n14:169522] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n10:176311] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n10:176311] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n10:176311] [11] [a35n07:138201] *** Process received signal *** +[a35n07:138201] Signal: Segmentation fault (11) +[a35n07:138201] Signal code: (3) +[a35n07:138201] Failing at address: 0xb170000000000a0 +[a35n07:138201] [ 0] [0x1000000504d8] +[a35n07:138201] [ 1] [b12n17:138142] *** Process received signal *** +[b12n17:138142] Signal: Segmentation fault (11) +[b12n17:138142] Signal code: (3) +[b12n17:138142] Failing at address: 0xb170000000000a0 +[b12n17:138142] [ 0] [0x1000000504d8] +[b12n17:138142] [ 1] [0x3ffffa4eb340] +[b12n17:138142] [ 2] [a27n17:15873] *** Process received signal *** +[a27n17:15873] Signal: Segmentation fault (11) +[a27n17:15873] Signal code: (3) +[a27n17:15873] Failing at address: 0x14170000000000a0 +[a27n17:15873] [ 0] [0x1000000504d8] +[a27n17:15873] [ 1] [0x3fffe425d930] +[a27n17:15873] [ 2] [b22n12:122963] *** Process received signal *** +[b22n12:122963] Signal: Segmentation fault (11) +[b22n12:122963] Signal code: (3) +[b22n12:122963] Failing at address: 0x14170000000000a0 +[b22n12:122963] [ 0] [0x1000000504d8] +[b22n12:122963] [b09n14:159239] *** Process received signal *** +[b09n14:159239] Signal: Segmentation fault (11) +[b09n14:159239] Signal code: (3) +[b09n14:159239] Failing at address: 0x5170000000000a0 +[b09n14:159239] [ 0] [0x1000000504d8] +[b09n14:159239] [ 1] [0x3fffcf1a0320] +[b09n14:159239] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n14:159239] [ 3] [b01n14:175715] *** Process received signal *** +[b01n14:175715] Signal: Segmentation fault (11) +[b01n14:175715] Signal code: (3) +[b01n14:175715] Failing at address: 0x14170000000000a0 +[b01n14:175715] [ 0] [0x1000000504d8] +[b01n14:175715] [ 1] [0x3ffff1874500] +[b01n14:175715] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n14:175715] [ 3] [b05n14:177511] *** Process received signal *** +[b05n14:177511] Signal: Segmentation fault (11) +[b05n14:177511] Signal code: (3) +[b05n14:177511] Failing at address: 0xb170000000000a0 +[b05n14:177511] [ 0] [0x1000000504d8] +[b05n14:177511] [ 1] [0x3fffceddf670] +[b05n14:177511] [ 2] [b03n06:01841] *** Process received signal *** +[b03n06:01841] Signal: Segmentation fault (11) +[b03n06:01841] Signal code: (3) +[b03n06:01841] Failing at address: 0x14170000000000a0 +[b03n06:01841] [ 0] [0x1000000504d8] +[b03n06:01841] [ 1] [0x3fffc6f90770] +[b03n06:01841] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n06:01841] [ 3] [b05n03:177927] *** Process received signal *** +[b05n03:177927] Signal: Segmentation fault (11) +[b05n03:177927] Signal code: (3) +[b05n03:177927] Failing at address: 0x14170000000000a0 +[b05n03:177927] [ 0] [0x1000000504d8] +[b05n03:177927] [ 1] [0x3ffffa2ab2f0] +[b05n03:177927] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n03:177927] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n01:141816] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n01:141816] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n01:141816] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n06:143011] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n06:143011] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n06:143011] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n16:124665] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n16:124665] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n11:171963] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n11:171963] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n08:08957] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n08:08957] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n08:08957] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n08:08957] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n08:147824] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n08:147824] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n08:147824] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n08:147824] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n08:147824] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n08:147824] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n08:147824] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n07:132090] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n07:132090] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n18:126908] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n18:126908] *** End of error message *** +[0x3fffc0d574a0] +[a26n10:67858] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n10:67858] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n10:67858] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n02:164493] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n04:169911] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n04:169911] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n04:169911] [13] [0x1000000504d8] +[a36n07:153177] [ 1] [0x3ffff2e33130] +[a36n07:153177] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n07:153177] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n07:153177] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n07:153177] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n13:156920] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n13:156920] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n13:156920] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n13:156920] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n13:156920] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n18:28221] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n18:28221] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n18:28221] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n18:28221] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n11:152987] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n15:85939] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n15:85939] [ 5] [b15n13:123708] *** Process received signal *** +[b15n13:123708] Signal: Segmentation fault (11) +[b15n13:123708] Signal code: (3) +[b15n13:123708] Failing at address: 0xb170000000000a0 +[b15n13:123708] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n16:01571] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n16:01571] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n16:01571] [ 6] [0x3fffee738a40] +[a28n03:10291] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n03:10291] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n03:10291] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n03:10291] [ 5] [b15n09:121754] [ 0] [0x1000000504d8] +[b15n09:121754] [ 1] [0x3fffd9369b10] +[b15n09:121754] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n09:121754] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n09:121754] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n09:173750] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n09:173750] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n09:173750] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n09:173750] [14] [b14n09:119548] *** Process received signal *** +[b14n09:119548] Signal: Segmentation fault (11) +[b14n09:119548] Signal code: (3) +[b14n09:119548] Failing at address: 0x14170000000000a0 +[b14n09:119548] [ 0] [0x1000000504d8] +[b14n09:119548] [ 1] [0x3fffd4a133c0] +[b14n09:119548] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n09:119548] [ 3] [b18n04:136995] *** Process received signal *** +[b18n04:136995] Signal: Segmentation fault (11) +[b18n04:136995] Signal code: (3) +[b18n04:136995] Failing at address: 0xb170000000000a0 +[b18n04:136995] [ 0] [0x1000000504d8] +[b18n04:136995] [ 1] [0x3ffff5216ce0] +[b18n04:136995] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n04:136995] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n11:01170] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n11:01170] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n11:01170] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n10:130582] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n10:130582] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n10:130582] [ 9] [0x1000000504d8] +[b12n11:137873] [ 1] [0x3ffffe13eca0] +[b12n11:137873] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n11:137873] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n11:137873] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n08:124726] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n08:124726] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n08:124726] [ 6] [b03n13:01298] *** Process received signal *** +[b03n13:01298] Signal: Segmentation fault (11) +[b03n13:01298] Signal code: (3) +[b03n13:01298] Failing at address: 0x5170000000000a0 +[b03n13:01298] [ 0] [0x1000000504d8] +[b03n13:01298] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n12:11141] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n12:11141] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n12:11141] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n16:153293] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n16:153293] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n16:153293] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n16:171919] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n16:171919] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n16:171919] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n16:171919] [ 8] [b04n15:177861] *** Process received signal *** +[b04n15:177861] Signal: Segmentation fault (11) +[b04n15:177861] Signal code: (3) +[b04n15:177861] Failing at address: 0x5170000000000a0 +[b04n15:177861] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n17:150932] *** End of error message *** +[b18n02:125370] *** Process received signal *** +[b18n02:125370] Signal: Segmentation fault (11) +[b18n02:125370] Signal code: (3) +[b18n02:125370] Failing at address: 0xb170000000000a0 +[b18n02:125370] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n15:55850] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n15:55850] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n15:55850] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n15:55850] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n15:55850] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n11:152843] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n11:152843] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n11:152843] [ 9] [b06n04:176000] *** Process received signal *** +[b06n04:176000] Signal: Segmentation fault (11) +[b06n04:176000] Signal code: (3) +[b06n04:176000] Failing at address: 0x14170000000000a0 +[b06n04:176000] [ 0] [0x1000000504d8] +[b06n04:176000] [ 1] [0x3ffff172bc50] +[b06n04:176000] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n04:176000] [ 3] [0x3fffdf2171c0] +[a33n15:148580] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n15:148580] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n15:148580] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n15:148580] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n02:162034] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n02:162034] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n02:162034] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n02:162034] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n02:162034] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n02:162034] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n02:162034] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n02:162034] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n02:162034] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n02:162034] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n08:174411] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n08:174411] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n08:174411] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n08:174411] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n08:174411] [ 8] [a31n01:173778] *** Process received signal *** +[a31n01:173778] Signal: Segmentation fault (11) +[a31n01:173778] Signal code: (3) +[a31n01:173778] Failing at address: 0xb170000000000a0 +[a31n01:173778] [ 0] [0x1000000504d8] +[a31n01:173778] [ 1] [a36n18:163106] *** Process received signal *** +[a36n18:163106] Signal: Segmentation fault (11) +[a36n18:163106] Signal code: (3) +[a36n18:163106] Failing at address: 0xb170000000000a0 +[b22n17:119583] *** Process received signal *** +[b22n17:119583] Signal: Segmentation fault (11) +[b22n17:119583] Signal code: (3) +[b22n17:119583] Failing at address: 0xb170000000000a0 +[b22n17:119583] [ 0] [0x1000000504d8] +[b22n17:119583] [ 1] [0x3fffcf083ea0] +[b22n17:119583] [ 2] [b14n16:122494] *** Process received signal *** +[b14n16:122494] Signal: Segmentation fault (11) +[b14n16:122494] Signal code: (3) +[b14n16:122494] Failing at address: 0x14170000000000a0 +[b14n16:122494] [ 0] [0x1000000504d8] +[b14n16:122494] [ 1] [a35n10:151892] *** Process received signal *** +[a35n10:151892] Signal: Segmentation fault (11) +[a35n10:151892] Signal code: (3) +[a35n10:151892] Failing at address: 0xb170000000000a0 +[a35n10:151892] [ 0] [0x1000000504d8] +[a35n10:151892] [ 1] [0x3fffe7dbc080] +[a35n10:151892] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n10:151892] [ 3] [b17n12:123233] *** Process received signal *** +[b17n12:123233] Signal: Segmentation fault (11) +[b17n12:123233] Signal code: (3) +[b17n12:123233] Failing at address: 0x14170000000000a0 +[b17n12:123233] [ 0] [0x1000000504d8] +[b17n12:123233] [ 1] [0x3fffde537110] +[b17n12:123233] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n12:123233] [ 3] [a26n12:52454] *** Process received signal *** +[a26n12:52454] Signal: Segmentation fault (11) +[a26n12:52454] Signal code: (3) +[a26n12:52454] Failing at address: 0x14170000000000a0 +[a26n12:52454] [ 0] [0x1000000504d8] +[a26n12:52454] [ 1] [0x3fffc25dfd40] +[a26n12:52454] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n12:52454] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n12:52454] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n12:148383] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n12:148383] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n12:148383] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n12:148383] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n12:148383] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n12:148383] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n12:148383] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n12:148383] [11] [b10n17:164559] *** Process received signal *** +[b10n17:164559] Signal: Segmentation fault (11) +[b10n17:164559] Signal code: (3) +[b10n17:164559] Failing at address: 0x14170000000000a0 +[b10n17:164559] [ 0] [0x1000000504d8] +[b10n17:164559] [ 1] [a31n12:156917] *** Process received signal *** +[a31n12:156917] Signal: Segmentation fault (11) +[a31n12:156917] Signal code: (3) +[a31n12:156917] Failing at address: 0x14170000000000a0 +[a31n12:156917] [ 0] [0x1000000504d8] +[a31n12:156917] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n11:128958] *** End of error message *** +[b06n14:173895] *** Process received signal *** +[b06n14:173895] Signal: Segmentation fault (11) +[b06n14:173895] Signal code: (3) +[b06n14:173895] Failing at address: 0x14170000000000a0 +[b06n14:173895] [ 0] [0x1000000504d8] +[b06n14:173895] [ 1] [ 0] [0x1000000504d8] +[b15n03:119829] [ 1] [0x3fffe97c1830] +[b15n03:119829] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n03:119829] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n03:119829] [ 4] [b10n08:129888] *** Process received signal *** +[b10n08:129888] Signal: Segmentation fault (11) +[b10n08:129888] Signal code: (3) +[b10n08:129888] Failing at address: 0x14170000000000a0 +[b10n08:129888] [ 0] [0x1000000504d8] +[b10n08:129888] [ 1] [0x3fffd08cdac0] +[b10n08:129888] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n08:129888] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n15:177821] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n15:177821] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n15:177821] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n15:177821] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n15:177821] *** End of error message *** +[b06n02:163321] *** Process received signal *** +[b06n02:163321] Signal: Segmentation fault (11) +[b06n02:163321] Signal code: (3) +[b06n02:163321] Failing at address: 0x14170000000000a0 +[b06n02:163321] [ 0] [0x1000000504d8] +[b06n02:163321] [ 1] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n09:152063] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n09:152063] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n09:152063] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n09:152063] *** End of error message *** +[b14n07:127355] *** Process received signal *** +[b14n07:127355] Signal: Segmentation fault (11) +[b14n07:127355] Signal code: (3) +[b14n07:127355] Failing at address: 0xb170000000000a0 +[b14n07:127355] [ 0] [b22n09:122068] *** Process received signal *** +[b22n09:122068] Signal: Segmentation fault (11) +[b22n09:122068] Signal code: (3) +[b22n09:122068] Failing at address: 0xb170000000000a0 +[b22n09:122068] [ 0] [0x1000000504d8] +[b22n09:122068] [ 1] [0x3fffd4447ad0] +[b22n09:122068] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n09:122068] [ 3] [b09n03:168210] *** Process received signal *** +[b09n03:168210] Signal: Segmentation fault (11) +[b09n03:168210] Signal code: (3) +[b09n03:168210] Failing at address: 0x14170000000000a0 +[b09n03:168210] [ 0] [0x1000000504d8] +[b09n03:168210] [ 1] [0x3fffc900fdc0] +[b09n03:168210] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n03:168210] [ 3] [b04n11:176299] *** Process received signal *** +[b04n11:176299] Signal: Segmentation fault (11) +[b04n11:176299] Signal code: (3) +[b04n11:176299] Failing at address: 0x14170000000000a0 +[b04n11:176299] [ 0] [0x1000000504d8] +[b04n11:176299] [ 1] [0x3fffd7b89110] +[b04n11:176299] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n11:176299] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n07:125449] *** End of error message *** +[b01n15:03733] *** Process received signal *** +[b01n15:03733] Signal: Segmentation fault (11) +[b01n15:03733] Signal code: (3) +[b01n15:03733] Failing at address: 0x14170000000000a0 +[a30n08:163101] *** Process received signal *** +[a30n08:163101] Signal: Segmentation fault (11) +[a30n08:163101] Signal code: (3) +[a30n08:163101] Failing at address: 0xb170000000000a0 +[a30n08:163101] [ 0] [0x1000000504d8] +[a30n08:163101] [b07n15:166706] *** Process received signal *** +[b07n15:166706] Signal: Segmentation fault (11) +[b07n15:166706] Signal code: (3) +[b07n15:166706] Failing at address: 0xb170000000000a0 +[b07n15:166706] [ 0] [0x1000000504d8] +[b07n15:166706] [ 1] [0x3fffd71e6da0] +[b07n15:166706] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n15:166706] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n15:166706] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n02:155384] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n02:155384] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n02:155384] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n02:155384] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n02:155384] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n02:155384] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n02:155384] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n02:155384] [14] [b01n16:166689] *** Process received signal *** +[b01n16:166689] Signal: Segmentation fault (11) +[b01n16:166689] Signal code: (3) +[b01n16:166689] Failing at address: 0xb170000000000a0 +[b01n16:166689] [ 0] [0x1000000504d8] +[b01n16:166689] [ 1] [b17n07:125472] *** Process received signal *** +[b17n07:125472] Signal: Segmentation fault (11) +[b17n07:125472] Signal code: (3) +[b17n07:125472] Failing at address: 0xb170000000000a0 +[b17n07:125472] [ 0] [b06n01:165166] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n13:163196] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n13:163196] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n13:163196] [13] [a34n08:163160] *** Process received signal *** +[a34n08:163160] Signal: Segmentation fault (11) +[a34n08:163160] Signal code: (3) +[a34n08:163160] Failing at address: 0xb170000000000a0 +[a34n08:163160] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n13:165263] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n13:165263] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n13:165263] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n13:165263] [14] [a31n08:156371] *** Process received signal *** +[a31n08:156371] Signal: Segmentation fault (11) +[a31n08:156371] Signal code: (3) +[a31n08:156371] Failing at address: 0x14170000000000a0 +[a31n08:156371] [a30n06:155088] *** Process received signal *** +[a30n06:155088] Signal: Segmentation fault (11) +[a30n06:155088] Signal code: (3) +[a30n06:155088] Failing at address: 0x14170000000000a0 +[a30n06:155088] [ 0] [0x1000000504d8] +[a30n06:155088] [ 1] [0x3fffe54b96b0] +[a30n06:155088] [ 2] [b03n11:175598] *** Process received signal *** +[b03n11:175598] Signal: Segmentation fault (11) +[b03n11:175598] Signal code: (3) +[b03n11:175598] Failing at address: 0x14170000000000a0 +[b03n11:175598] [ 0] [0x1000000504d8] +[b03n11:175598] [ 1] [0x3fffe6a53920] +[b03n11:175598] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n11:175598] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n11:175598] [ 4] [a30n15:154385] *** Process received signal *** +[a30n15:154385] Signal: Segmentation fault (11) +[a30n15:154385] Signal code: (3) +[a30n15:154385] Failing at address: 0x14170000000000a0 +[a30n15:154385] [ 0] [b10n04:169411] *** Process received signal *** +[b10n04:169411] Signal: Segmentation fault (11) +[b10n04:169411] Signal code: (3) +[b10n04:169411] Failing at address: 0xb170000000000a0 +[b10n04:169411] [ 0] [0x1000000504d8] +[b10n04:169411] [ 1] [0x3ffffff681c0] +[b10n04:169411] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n04:169411] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n04:169411] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n04:169411] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n04:169411] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n04:169411] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n04:169411] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n04:169411] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n04:169411] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n04:169411] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n04:169411] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n04:169411] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n04:169411] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n04:169411] *** End of error message *** +[0x1000000504d8] +[b23n12:115636] [ 1] [0x3fffdf1e6aa0] +[b23n12:115636] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n12:115636] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n12:115636] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n12:115636] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n05:169787] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n05:169787] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n05:169787] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n05:169787] [ 8] [b22n11:120241] *** Process received signal *** +[b22n11:120241] Signal: Segmentation fault (11) +[b22n11:120241] Signal code: (3) +[b22n11:120241] Failing at address: 0x14170000000000a0 +[b22n11:120241] [ 0] [0x1000000504d8] +[b22n11:120241] [ 1] [0x3ffffd1901c0] +[b22n11:120241] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n11:120241] [ 3] [b11n12:154342] *** Process received signal *** +[b11n12:154342] Signal: Segmentation fault (11) +[b11n12:154342] Signal code: (3) +[b11n12:154342] Failing at address: 0xb170000000000a0 +[b11n12:154342] [ 0] [0x1000000504d8] +[b11n12:154342] [ 1] [0x3fffefbb19b0] +[b11n12:154342] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n12:154342] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n12:154342] [ 4] [b04n06:10224] [ 0] [0x1000000504d8] +[b04n06:10224] [ 1] [0x3fffd26b9200] +[b04n06:10224] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n06:10224] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n06:10224] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n06:177848] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n06:177848] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n06:177848] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n06:177848] [ 8] [b17n15:137008] *** Process received signal *** +[b17n15:137008] Signal: Segmentation fault (11) +[b17n15:137008] Signal code: (3) +[b17n15:137008] Failing at address: 0xb170000000000a0 +[b17n15:137008] [ 0] [0x1000000504d8] +[b17n15:137008] [ 1] [0x3fffe58d43c0] +[b17n15:137008] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n15:137008] [ 3] [b14n15:123041] *** Process received signal *** +[b14n15:123041] Signal: Segmentation fault (11) +[b14n15:123041] Signal code: (3) +[b14n15:123041] Failing at address: 0x14170000000000a0 +[b14n15:123041] [ 0] [0x1000000504d8] +[b14n15:123041] [ 1] [0x3fffed51c590] +[b14n15:123041] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n15:123041] [ 3] [0x1000000504d8] +[a35n17:146798] [ 1] [0x3fffc287c940] +[a35n17:146798] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n17:146798] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n17:146798] [ 4] [0x1000000504d8] +[b13n12:140120] [ 1] [0x3fffc2bf1e80] +[b13n12:140120] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n12:140120] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n12:140120] [ 4] [a32n11:152815] *** Process received signal *** +[a32n11:152815] Signal: Segmentation fault (11) +[a32n11:152815] Signal code: (3) +[a32n11:152815] Failing at address: 0x14170000000000a0 +[a32n11:152815] [ 0] [0x1000000504d8] +[a32n11:152815] [ 1] [0x3fffc5287dd0] +[a32n11:152815] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n11:152815] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n11:152815] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n11:152815] [ 5] [b10n03:168930] *** Process received signal *** +[b10n03:168930] Signal: Segmentation fault (11) +[b10n03:168930] Signal code: (3) +[b10n03:168930] Failing at address: 0xb170000000000a0 +[b10n03:168930] [ 0] [0x1000000504d8] +[b10n03:168930] [ 1] [0x3fffef984d60] +[b10n03:168930] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n03:168930] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n03:168930] [ 4] [b14n17:119467] *** Process received signal *** +[b14n17:119467] Signal: Segmentation fault (11) +[b14n17:119467] Signal code: (3) +[b14n17:119467] Failing at address: 0x14170000000000a0 +[b14n17:119467] [ 0] [0x1000000504d8] +[b14n17:119467] [ 1] [0x3ffff3eedcc0] +[b14n17:119467] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n17:119467] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n02:49107] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n02:49107] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n02:49107] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n14:177224] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n14:177224] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n14:177224] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n11:02686] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n12:11420] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n12:11420] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n12:11420] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n12:11420] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n12:11420] [ 8] [0x1000000504d8] +[b15n17:126526] [ 1] [0x3fffc4f945f0] +[b15n17:126526] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n17:126526] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n17:126526] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n17:126526] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n14:163548] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n14:163548] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n14:163548] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n14:163548] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n14:163548] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n15:161758] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n15:161758] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n15:161758] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n15:161758] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n13:06927] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n13:06927] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n13:06927] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n13:06927] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n13:06927] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n16:124798] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n16:124798] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n16:124798] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n09:147792] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n09:147792] *** End of error message *** +[b17n02:127220] *** Process received signal *** +[b17n02:127220] Signal: Segmentation fault (11) +[b17n02:127220] Signal code: (3) +[b17n02:127220] Failing at address: 0x14170000000000a0 +[b17n02:127220] [ 0] [0x1000000504d8] +[b17n02:127220] [ 1] [0x3fffd65b6170] +[b17n02:127220] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n02:127220] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n17:163817] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n17:163817] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n17:163817] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n17:163817] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n02:01432] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n02:01432] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n02:01432] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n02:01432] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n02:01432] [13] [a30n03:167778] *** Process received signal *** +[a30n03:167778] Signal: Segmentation fault (11) +[a30n03:167778] Signal code: (3) +[a30n03:167778] Failing at address: 0x14170000000000a0 +[a30n03:167778] [ 0] [0x1000000504d8] +[a30n03:167778] [ 1] [0x3fffc7cc1150] +[a30n03:167778] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n03:125240] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n03:125240] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n06:154622] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n06:154622] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n11:148673] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n11:148673] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n11:148673] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n11:148673] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n05:162565] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n05:162565] [ 6] [a34n01:164963] *** Process received signal *** +[a34n01:164963] Signal: Segmentation fault (11) +[a34n01:164963] Signal code: (3) +[a34n01:164963] Failing at address: 0x14170000000000a0 +[a34n01:164963] [ 0] [0x3fffe32c4020] +[b01n08:172427] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n08:172427] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n08:172427] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n06:130544] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n06:130544] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n06:130544] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n02:122557] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n02:122557] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n02:122557] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n02:122557] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n03:151881] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n03:151881] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n03:151881] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n11:164996] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n11:164996] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n11:164996] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n11:164996] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n11:164996] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n10:161720] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n10:161720] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n10:161720] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n10:161720] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n10:161720] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n10:161720] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n10:161720] [10] [a28n08:175806] *** Process received signal *** +[a28n08:175806] Signal: Segmentation fault (11) +[a28n08:175806] Signal code: (3) +[a28n08:175806] Failing at address: 0x5170000000000a0 +[a28n08:175806] [ 0] [0x1000000504d8] +[a28n08:175806] [ 1] [0x3fffc4d97cd0] +[a28n08:175806] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n08:175806] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n08:175806] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n08:175806] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n08:175806] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n08:175806] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n08:175806] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n08:175806] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n08:175806] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n08:175806] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n08:175806] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n08:175806] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n08:175806] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n08:175806] *** End of error message *** +[b10n06:162920] *** Process received signal *** +[b10n06:162920] Signal: Segmentation fault (11) +[b10n06:162920] Signal code: (3) +[b10n06:162920] Failing at address: 0xb170000000000a0 +[b10n06:162920] [ 0] [0x1000000504d8] +[b10n06:162920] [ 1] [0x3fffc6f4c0d0] +[b10n06:162920] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n06:162920] [ 3] [a29n10:164940] [ 0] [0x1000000504d8] +[a29n10:164940] [ 1] [0x3fffc9487e00] +[a29n10:164940] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n10:164940] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n01:171788] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n01:171788] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n01:171788] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n01:171788] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n01:171788] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n01:171788] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n01:171788] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n12:165844] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n12:165844] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n12:165844] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n12:165844] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n12:165844] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n12:165844] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n12:165844] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n12:165844] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n12:165844] *** End of error message *** +[b15n16:121928] *** Process received signal *** +[b15n16:121928] Signal: Segmentation fault (11) +[b15n16:121928] Signal code: (3) +[b15n16:121928] Failing at address: 0x14170000000000a0 +[b15n16:121928] [ 0] [0x1000000504d8] +[b15n16:121928] [ 1] [0x3fffde25a960] +[b15n16:121928] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n16:121928] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n09:106789] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n09:106789] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n09:106789] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n09:106789] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n09:106789] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n09:106789] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n09:106789] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n09:106789] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n09:106789] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n08:164751] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n08:164751] [ 6] [b21n09:120693] *** Process received signal *** +[b21n09:120693] Signal: Segmentation fault (11) +[b21n09:120693] Signal code: (3) +[b21n09:120693] Failing at address: 0xb170000000000a0 +[b21n09:120693] [ 0] [0x1000000504d8] +[b21n09:120693] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n12:134494] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n12:134494] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n12:134494] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n12:134494] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n12:134494] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n12:134494] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n12:134494] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n12:134494] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n12:134494] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n12:134494] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n12:134494] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n05:120331] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n05:120331] *** End of error message *** +[0x1000000504d8] +[b06n06:163677] [ 1] [0x3fffc2768a50] +[b06n06:163677] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n06:163677] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n06:163677] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n05:173082] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n05:173082] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n05:173082] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n05:173082] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n06:127277] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n06:127277] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n06:127277] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n06:127277] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n06:127277] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n06:127277] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n06:127277] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n16:164551] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n16:164551] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n16:164551] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n07:60173] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n07:60173] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n07:60173] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n07:60173] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n07:60173] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n07:60173] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n07:60173] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n07:60173] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n07:60173] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n07:60173] *** End of error message *** +[b18n08:131378] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n03:175066] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n03:175066] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n03:175066] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n03:175066] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n03:175066] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n03:175066] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n03:175066] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n03:175066] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n03:175066] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n03:175066] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n03:175066] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n03:175066] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n06:161721] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n06:161721] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n06:161721] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n06:161721] [ 6] [a35n06:138766] *** Process received signal *** +[a35n06:138766] Signal: Segmentation fault (11) +[a35n06:138766] Signal code: (3) +[a35n06:138766] Failing at address: 0x14170000000000a0 +[a35n06:138766] [ 0] [0x1000000504d8] +[a35n06:138766] [ 1] [0x3fffc5b2b460] +[a35n06:138766] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n06:138766] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n06:138766] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n09:155756] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n09:155756] [ 5] [a30n17:176413] *** Process received signal *** +[a30n17:176413] Signal: Segmentation fault (11) +[a30n17:176413] Signal code: (3) +[a30n17:176413] Failing at address: 0x14170000000000a0 +[a30n17:176413] [ 0] [0x1000000504d8] +[a30n17:176413] [ 1] [0x3ffffcd35c80] +[a30n17:176413] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n17:176413] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n17:123238] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n17:123238] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n17:123238] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n17:168156] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n17:168156] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n17:168156] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n17:168156] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n17:169738] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n17:169738] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n17:169738] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n17:169738] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n17:169738] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n17:169738] [ 8] [b08n18:164010] *** Process received signal *** +[b08n18:164010] Signal: Segmentation fault (11) +[b08n18:164010] Signal code: (3) +[b08n18:164010] Failing at address: 0xb170000000000a0 +[b08n18:164010] [ 0] [0x1000000504d8] +[b08n18:164010] [ 1] [0x3fffbffffa50] +[b08n18:164010] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n18:164010] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n18:164010] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n18:164010] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n18:164010] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n18:164010] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n18:164010] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n18:164010] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n18:164010] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n18:164010] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n18:164010] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n18:164010] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n18:164010] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n18:164010] *** End of error message *** +[0x1000000504d8] +[b10n01:165124] [ 1] [0x3fffdb0f4910] +[b10n01:165124] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n01:165124] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n01:165124] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n01:165124] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n07:152085] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n07:152085] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n07:152085] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n03:154295] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n03:154295] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n03:154295] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n06:117525] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n06:117525] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n06:117525] [ 9] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n01:123574] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n01:123574] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n01:123574] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n01:123574] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n01:123574] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n12:138738] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n12:138738] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n12:138738] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n11:117476] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n11:117476] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n11:117476] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n11:117476] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n06:157791] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n06:157791] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n06:157791] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n06:157791] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n06:157791] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n06:157791] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n06:157791] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n06:157791] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n12:141260] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n12:141260] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n12:141260] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n12:141260] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n12:141260] [ 8] [b01n12:178597] *** Process received signal *** +[b01n12:178597] Signal: Segmentation fault (11) +[b01n12:178597] Signal code: (3) +[b01n12:178597] Failing at address: 0xb170000000000a0 +[b01n12:178597] [ 0] [0x1000000504d8] +[b01n12:178597] [ 1] [0x3fffd135ec50] +[b01n12:178597] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n12:178597] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n04:158633] *** End of error message *** +[b08n14:178327] [ 0] [0x1000000504d8] +[b08n14:178327] [ 1] [0x3fffed590e20] +[b08n14:178327] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n14:178327] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n14:178327] [ 4] [b08n07:168962] *** Process received signal *** +[b08n07:168962] Signal: Segmentation fault (11) +[b08n07:168962] Signal code: (3) +[b08n07:168962] Failing at address: 0x14170000000000a0 +[b08n07:168962] [ 0] [0x1000000504d8] +[b08n07:168962] [ 1] [0x3fffd64f3820] +[b08n07:168962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n07:168962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n07:168962] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n17:176828] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n17:176828] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n10:16975] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n10:16975] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n10:16975] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n10:16975] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n10:16975] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n10:16975] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n10:16975] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n10:16975] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n10:16975] [13] [0x3fffd5aa51f0] +[b23n09:119635] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n09:119635] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n09:119635] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n09:119635] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n04:145750] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n04:145750] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n11:120580] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n11:120580] [ 6] [0x3fffda1bbe30] +[b02n13:171110] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n13:171110] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n13:171110] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n13:171110] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n13:171110] [ 6] [a33n05:139433] *** Process received signal *** +[a33n05:139433] Signal: Segmentation fault (11) +[a33n05:139433] Signal code: (3) +[a33n05:139433] Failing at address: 0xb170000000000a0 +[a33n05:139433] [ 0] [0x1000000504d8] +[a33n05:139433] [ 1] [0x3fffdf00dfa0] +[a33n05:139433] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n05:139433] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n10:11488] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n10:11488] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n10:11488] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n07:160803] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n07:160803] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n07:160803] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n03:46687] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n03:46687] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n03:46687] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n17:177404] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n17:177404] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n17:177404] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n17:177404] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n13:153679] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n13:153679] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n13:153679] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n13:153679] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n13:153679] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n13:153679] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n16:148189] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n16:148189] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n16:148189] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n16:148189] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n16:148189] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n16:148189] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n16:148189] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n16:148189] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n16:148189] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n11:03987] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n11:03987] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n03:120671] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n03:120671] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n03:120671] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n03:120671] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n03:120671] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n03:120671] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n03:120671] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n03:120671] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n03:120671] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n03:120671] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n03:120671] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n16:08385] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n16:08385] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n16:08385] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n17:119978] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n17:119978] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n17:119978] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n18:165697] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n18:165697] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n18:165697] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n18:165697] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n18:165697] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n18:165697] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n18:165697] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n18:165697] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n18:165697] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n18:165697] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n05:123938] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n05:123938] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n05:123938] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n05:123938] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n05:123938] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n05:123938] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n05:123938] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n05:123938] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n05:123938] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n05:123938] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n05:123938] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n05:123938] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n11:134522] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n11:134522] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n11:134522] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n11:134522] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n04:44755] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n04:44755] [ 6] [a28n14:08398] [ 0] [0x1000000504d8] +[a28n14:08398] [ 1] [0x3fffdc69a8e0] +[a28n14:08398] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n14:08398] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n05:139910] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n05:139910] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n18:179168] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n18:179168] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n18:179168] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n18:179168] [11] [b03n03:06824] *** Process received signal *** +[b03n03:06824] Signal: Segmentation fault (11) +[b03n03:06824] Signal code: (3) +[b03n03:06824] Failing at address: 0x14170000000000a0 +[b03n03:06824] [ 0] [0x1000000504d8] +[b03n03:06824] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n12:123559] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n12:123559] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n12:123559] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n13:123054] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n13:123054] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n13:123054] [ 5] [a30n05:156009] *** Process received signal *** +[a30n05:156009] Signal: Segmentation fault (11) +[a30n05:156009] Signal code: (3) +[a30n05:156009] Failing at address: 0x14170000000000a0 +[a30n05:156009] [ 0] [0x1000000504d8] +[a30n05:156009] [ 1] [0x3fffe4eb79a0] +[a30n05:156009] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n05:156009] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n05:156009] [ 4] [b17n13:123516] *** Process received signal *** +[b17n13:123516] Signal: Segmentation fault (11) +[b17n13:123516] Signal code: (3) +[b17n13:123516] Failing at address: 0x14170000000000a0 +[b17n13:123516] [ 0] [0x1000000504d8] +[b17n13:123516] [ 1] [0x3fffdaacf780] +[b17n13:123516] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n13:123516] [ 3] [a30n13:154235] *** Process received signal *** +[a30n13:154235] Signal: Segmentation fault (11) +[a30n13:154235] Signal code: (3) +[a30n13:154235] Failing at address: 0x14170000000000a0 +[a30n13:154235] [ 0] [0x1000000504d8] +[a30n13:154235] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n07:165009] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n07:165009] *** End of error message *** +[a26n16:53796] *** Process received signal *** +[a26n16:53796] Signal: Segmentation fault (11) +[a26n16:53796] Signal code: (3) +[a26n16:53796] Failing at address: 0xb170000000000a0 +[a26n16:53796] [ 0] [0x1000000504d8] +[a26n16:53796] [ 1] [a30n10:169211] *** Process received signal *** +[a30n10:169211] Signal: Segmentation fault (11) +[a30n10:169211] Signal code: (3) +[a30n10:169211] Failing at address: 0x14170000000000a0 +[a30n10:169211] [ 0] [0x1000000504d8] +[a30n10:169211] [ 1] [0x3fffe449abe0] +[a30n10:169211] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n10:169211] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n10:169211] [ 4] [a29n05:174781] *** Process received signal *** +[a29n05:174781] Signal: Segmentation fault (11) +[a29n05:174781] Signal code: (3) +[a29n05:174781] Failing at address: 0x14170000000000a0 +[a29n05:174781] [ 0] [0x1000000504d8] +[a29n05:174781] [ 1] [0x3fffdc4b9dd0] +[a29n05:174781] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n05:174781] [ 3] [a30n04:03149] *** Process received signal *** +[a30n04:03149] Signal: Segmentation fault (11) +[a30n04:03149] Signal code: (3) +[a30n04:03149] Failing at address: 0x14170000000000a0 +[a30n04:03149] [ 0] [0x1000000504d8] +[a30n04:03149] [ 1] [0x3fffdfa36fa0] +[a30n04:03149] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n04:03149] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n04:03149] [ 4] [b09n08:176679] *** Process received signal *** +[b09n08:176679] Signal: Segmentation fault (11) +[b09n08:176679] Signal code: (3) +[b09n08:176679] Failing at address: 0xb170000000000a0 +[b12n08:141873] *** Process received signal *** +[b12n08:141873] Signal: Segmentation fault (11) +[b12n08:141873] Signal code: (3) +[b12n08:141873] Failing at address: 0xb170000000000a0 +[b12n08:141873] [ 0] [0x1000000504d8] +[b12n08:141873] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n01:163389] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n01:163389] [ 6] [b07n12:160964] *** Process received signal *** +[b07n12:160964] Signal: Segmentation fault (11) +[b07n12:160964] Signal code: (3) +[b07n12:160964] Failing at address: 0x14170000000000a0 +[b07n12:160964] [ 0] [b15n14:133965] *** Process received signal *** +[b15n14:133965] Signal: Segmentation fault (11) +[b15n14:133965] Signal code: (3) +[b15n14:133965] Failing at address: 0x14170000000000a0 +[b15n14:133965] [ 0] [0x1000000504d8] +[b15n14:133965] [ 1] [0x3fffe6b34720] +[b15n14:133965] [ 2] [b11n15:166120] *** Process received signal *** +[b11n15:166120] Signal: Segmentation fault (11) +[b11n15:166120] Signal code: (3) +[b11n15:166120] Failing at address: 0x14170000000000a0 +[b11n15:166120] [ 0] [0x1000000504d8] +[b11n15:166120] [ 1] [a34n03:151707] *** Process received signal *** +[a34n03:151707] Signal: Segmentation fault (11) +[a34n03:151707] Signal code: (3) +[a34n03:151707] Failing at address: 0x14170000000000a0 +[a34n03:151707] [ 0] [0x1000000504d8] +[a34n03:151707] [ 1] [0x3fffde256a90] +[a34n03:151707] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n03:151707] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n03:151707] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n03:151707] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n03:151707] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n03:151707] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n03:151707] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n03:151707] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n03:151707] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n03:151707] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n03:151707] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n03:151707] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n03:151707] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n03:151707] *** End of error message *** +[b22n10:118387] *** Process received signal *** +[b22n10:118387] Signal: Segmentation fault (11) +[b22n10:118387] Signal code: (3) +[b22n10:118387] Failing at address: 0xb170000000000a0 +[b22n10:118387] [ 0] [0x1000000504d8] +[b22n10:118387] [ 1] [0x3fffeb7fc980] +[b22n10:118387] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n10:118387] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n10:118387] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n10:118387] [ 5] [a31n16:162629] *** Process received signal *** +[a31n16:162629] Signal: Segmentation fault (11) +[a31n16:162629] Signal code: (3) +[a31n16:162629] Failing at address: 0x14170000000000a0 +[a31n16:162629] [ 0] [0x1000000504d8] +[a31n16:162629] [ 1] [0x3fffff62f5c0] +[a31n16:162629] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n16:162629] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n06:179452] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n06:179452] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n06:179452] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n06:179452] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n06:179452] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n06:179452] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n06:179452] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n06:179452] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n06:179452] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n06:179452] [13] [b04n01:96751] *** Process received signal *** +[b04n01:96751] Signal: Segmentation fault (11) +[b04n01:96751] Signal code: (3) +[b04n01:96751] Failing at address: 0x5170000000000a0 +[b04n01:96751] [ 0] [0x1000000504d8] +[b04n01:96751] [ 1] [0x3ffff6847ab0] +[b04n01:96751] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n01:96751] [ 3] [b13n18:128540] *** Process received signal *** +[b13n18:128540] Signal: Segmentation fault (11) +[b13n18:128540] Signal code: (3) +[b13n18:128540] Failing at address: 0xb170000000000a0 +[b13n18:128540] [ 0] [0x1000000504d8] +[b13n18:128540] [ 1] [0x3fffda406c20] +[b13n18:128540] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n18:128540] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n18:128540] [ 4] [b04n09:12247] *** Process received signal *** +[b04n09:12247] Signal: Segmentation fault (11) +[b04n09:12247] Signal code: (3) +[b04n09:12247] Failing at address: 0x14170000000000a0 +[b04n09:12247] [ 0] [0x1000000504d8] +[b04n09:12247] [ 1] [0x3fffc7fb9a70] +[b04n09:12247] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n09:12247] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n08:116901] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n08:116901] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n08:116901] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n08:116901] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n08:116901] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n08:116901] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n08:116901] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n08:116901] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n08:116901] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n08:116901] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n08:116901] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n08:116901] *** End of error message *** +[0x3ffff53a6a10] +[b14n11:130807] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n11:130807] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n11:130807] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n11:130807] [ 5] [b03n01:170765] *** Process received signal *** +[b03n01:170765] Signal: Segmentation fault (11) +[b03n01:170765] Signal code: (3) +[b03n01:170765] Failing at address: 0xb170000000000a0 +[b03n01:170765] [ 0] [0x1000000504d8] +[b03n01:170765] [ 1] [0x3ffffd264a70] +[b03n01:170765] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n01:170765] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n01:170765] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n08:159021] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n08:159021] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n08:159021] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n08:159021] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n08:159021] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n08:159021] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n08:159021] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n18:123413] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n18:123413] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n18:123413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n18:123413] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n18:123413] [ 9] [a33n10:140412] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n10:140412] [10] [b14n08:127446] *** Process received signal *** +[b14n08:127446] Signal: Segmentation fault (11) +[b14n08:127446] Signal code: (3) +[b14n08:127446] Failing at address: 0x14170000000000a0 +[b14n08:127446] [ 0] [0x1000000504d8] +[b14n08:127446] [ 1] [0x3fffe94b6230] +[a28n05:04521] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n05:04521] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n05:04521] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n05:04521] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n16:154185] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n16:154185] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n16:154185] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n16:154185] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n12:145500] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n12:145500] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n12:145500] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n10:131275] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n10:131275] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n10:131275] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n10:131275] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n07:140135] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n07:140135] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n07:140135] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n07:140135] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n07:140135] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n07:140135] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n07:140135] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n07:140135] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n07:140135] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n07:140135] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n07:140135] [14] [0x1000000504d8] +[b18n15:120133] [ 1] [0x3fffe0afbf60] +[b18n15:120133] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n15:120133] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n15:120133] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n15:120133] [ 5] [b07n03:172648] *** Process received signal *** +[b07n03:172648] Signal: Segmentation fault (11) +[b07n03:172648] Signal code: (3) +[b07n03:172648] Failing at address: 0xb170000000000a0 +[b07n03:172648] [ 0] [0x1000000504d8] +[b07n03:172648] [ 1] [0x3fffc46868a0] +[b07n03:172648] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n03:172648] [ 3] [a32n17:138615] *** Process received signal *** +[a32n17:138615] Signal: Segmentation fault (11) +[a32n17:138615] Signal code: (3) +[a32n17:138615] Failing at address: 0x14170000000000a0 +[a32n17:138615] [ 0] [0x1000000504d8] +[a32n17:138615] [ 1] [0x3fffcfd0a3c0] +[a32n17:138615] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n17:138615] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n05:120230] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n05:120230] [ 5] [b13n13:133623] *** Process received signal *** +[b13n13:133623] Signal: Segmentation fault (11) +[b13n13:133623] Signal code: (3) +[b13n13:133623] Failing at address: 0x14170000000000a0 +[b13n13:133623] [ 0] [0x1000000504d8] +[b13n13:133623] [ 1] [0x3fffe9b10120] +[b13n13:133623] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n13:133623] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n13:133623] [ 4] [a33n18:149116] *** Process received signal *** +[a33n18:149116] Signal: Segmentation fault (11) +[a33n18:149116] Signal code: (3) +[a33n18:149116] Failing at address: 0x14170000000000a0 +[a33n18:149116] [ 0] [0x1000000504d8] +[a33n18:149116] [ 1] [0x3fffd3e67130] +[a33n18:149116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n18:149116] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n10:126613] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n10:126613] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n10:126613] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n10:126613] [ 8] [b15n15:122082] *** Process received signal *** +[b15n15:122082] Signal: Segmentation fault (11) +[b15n15:122082] Signal code: (3) +[b15n15:122082] Failing at address: 0x5170000000000a0 +[b15n15:122082] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n06:119116] *** End of error message *** +[a30n18:10721] *** Process received signal *** +[a30n18:10721] Signal: Segmentation fault (11) +[a30n18:10721] Signal code: (3) +[a30n18:10721] Failing at address: 0x14170000000000a0 +[a30n18:10721] [ 0] [0x1000000504d8] +[a30n18:10721] [ 1] [0x3fffc122ab60] +[a30n18:10721] [ 2] [b07n16:166495] *** Process received signal *** +[b07n16:166495] Signal: Segmentation fault (11) +[b07n16:166495] Signal code: (3) +[b07n16:166495] Failing at address: 0xb170000000000a0 +[b07n16:166495] [ 0] [0x1000000504d8] +[b07n16:166495] [ 1] [0x3fffc5c42780] +[b07n16:166495] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n16:166495] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n16:166495] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n16:166495] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n16:166495] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n16:166495] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n16:166495] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n16:166495] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n16:166495] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n16:166495] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n16:166495] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n16:166495] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n16:166495] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n16:166495] *** End of error message *** +[b02n10:01249] *** Process received signal *** +[b02n10:01249] Signal: Segmentation fault (11) +[b02n10:01249] Signal code: (3) +[b02n10:01249] Failing at address: 0x14170000000000a0 +[b02n10:01249] [ 0] [0x1000000504d8] +[b02n10:01249] [ 1] [0x3ffff2ec6920] +[b02n10:01249] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n10:01249] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n10:01249] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n10:01249] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n10:01249] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n10:01249] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n10:01249] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n10:01249] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n10:01249] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n10:01249] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n10:01249] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n10:01249] [13] [b17n01:124277] *** Process received signal *** +[b17n01:124277] Signal: Segmentation fault (11) +[b17n01:124277] Signal code: (3) +[b17n01:124277] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n11:156153] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n11:156153] *** End of error message *** +[b04n08:176176] *** Process received signal *** +[b04n08:176176] Signal: Segmentation fault (11) +[b04n08:176176] Signal code: (3) +[b04n08:176176] Failing at address: 0x14170000000000a0 +[b04n08:176176] [ 0] [0x1000000504d8] +[b04n08:176176] [ 1] [0x3ffff7712790] +[b04n08:176176] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n08:176176] [ 3] [b20n02:122178] *** Process received signal *** +[b20n02:122178] Signal: Segmentation fault (11) +[b20n02:122178] Signal code: (3) +[b20n02:122178] Failing at address: 0xb170000000000a0 +[b20n02:122178] [ 0] [0x1000000504d8] +[b20n02:122178] [ 1] [0x3fffff2e0dd0] +[b20n02:122178] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n02:122178] [ 3] [b13n15:139746] *** Process received signal *** +[b13n15:139746] Signal: Segmentation fault (11) +[b13n15:139746] Signal code: (3) +[b13n15:139746] Failing at address: 0x14170000000000a0 +[b13n15:139746] [ 0] [b06n09:168247] *** Process received signal *** +[b06n09:168247] Signal: Segmentation fault (11) +[b06n09:168247] Signal code: (3) +[b06n09:168247] Failing at address: 0x14170000000000a0 +[b06n09:168247] [ 0] [0x1000000504d8] +[b06n09:168247] [ 1] [0x3fffe9336380] +[b06n09:168247] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n09:168247] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n09:168247] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n09:168247] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n09:168247] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n09:168247] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n09:168247] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n09:168247] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n09:168247] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n09:168247] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n09:168247] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n09:168247] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n09:168247] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n09:168247] *** End of error message *** +[b03n09:173061] *** Process received signal *** +[b03n09:173061] Signal: Segmentation fault (11) +[b03n09:173061] Signal code: (3) +[b03n09:173061] Failing at address: 0xb170000000000a0 +[b03n09:173061] [ 0] [0x1000000504d8] +[b03n09:173061] [ 1] [0x3ffff798a6d0] +[b03n09:173061] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n09:173061] [ 3] [b13n14:126464] *** Process received signal *** +[b13n14:126464] Signal: Segmentation fault (11) +[b13n14:126464] Signal code: (3) +[b13n14:126464] Failing at address: 0xb170000000000a0 +[b13n14:126464] [ 0] [0x1000000504d8] +[b13n14:126464] [ 1] [a02n09:160374] [ 0] [0x1000000504d8] +[a02n09:160374] [ 1] [0x3fffec03e990] +[a02n09:160374] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n09:160374] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n09:160374] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n09:160374] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n09:160374] [ 6] [b13n06:133798] *** Process received signal *** +[b13n06:133798] Signal: Segmentation fault (11) +[b13n06:133798] Signal code: (3) +[b13n06:133798] Failing at address: 0x14170000000000a0 +[b13n06:133798] [ 0] [0x1000000504d8] +[b13n06:133798] [ 1] [0x3fffd46efcc0] +[b13n06:133798] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n06:133798] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n06:133798] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n06:133798] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n06:133798] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n06:133798] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n06:133798] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n06:133798] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n06:133798] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n06:133798] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n06:133798] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n06:133798] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n06:133798] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n06:133798] *** End of error message *** +[a26n14:51451] *** Process received signal *** +[a26n14:51451] Signal: Segmentation fault (11) +[a26n14:51451] Signal code: (3) +[a26n14:51451] Failing at address: 0x14170000000000a0 +[a26n14:51451] [ 0] [0x1000000504d8] +[a26n14:51451] [ 1] [0x3fffd7872400] +[a26n14:51451] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n14:51451] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n14:51451] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n14:51451] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n13:164692] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n13:164692] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n13:164692] *** End of error message *** +[a34n07:165092] *** Process received signal *** +[a34n07:165092] Signal: Segmentation fault (11) +[a34n07:165092] Signal code: (3) +[a34n07:165092] Failing at address: 0x14170000000000a0 +[a34n07:165092] [ 0] [0x1000000504d8] +[a34n07:165092] [ 1] [0x3fffebaf3b40] +[a34n07:165092] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n07:165092] [ 3] [b05n10:10923] *** Process received signal *** +[b05n10:10923] Signal: Segmentation fault (11) +[b05n10:10923] Signal code: (3) +[b05n10:10923] Failing at address: 0xb170000000000a0 +[b05n10:10923] [ 0] [0x1000000504d8] +[b05n10:10923] [ 1] [0x3fffc09a3c70] +[b05n10:10923] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n10:10923] [ 3] [b04n03:09642] *** Process received signal *** +[b04n03:09642] Signal: Segmentation fault (11) +[b04n03:09642] Signal code: (3) +[b04n03:09642] Failing at address: 0xb170000000000a0 +[b04n03:09642] [ 0] [0x1000000504d8] +[b04n03:09642] [ 1] [0x3fffc72b7720] +[b04n03:09642] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n03:09642] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n03:09642] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n03:09642] [ 5] [0x1000000504d8] +[b19n10:120976] [ 1] [0x3ffff5e32850] +[b19n10:120976] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n10:120976] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n10:120976] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n10:120976] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n10:120976] [ 6] [b05n05:173317] *** Process received signal *** +[b05n05:173317] Signal: Segmentation fault (11) +[b05n05:173317] Signal code: (3) +[b05n05:173317] Failing at address: 0x14170000000000a0 +[b05n05:173317] [ 0] [0x1000000504d8] +[b05n05:173317] [ 1] [0x3ffff9c37390] +[b05n05:173317] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n05:173317] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n10:140991] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n10:140991] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n10:140991] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n10:140991] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n10:140991] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n10:140991] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n10:140991] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n10:140991] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n10:140991] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n10:140991] *** End of error message *** +[b05n11:176582] *** Process received signal *** +[b05n11:176582] Signal: Segmentation fault (11) +[b05n11:176582] Signal code: (3) +[b05n11:176582] Failing at address: 0xb170000000000a0 +[b05n11:176582] [ 0] [0x1000000504d8] +[b05n11:176582] [ 1] [0x3fffc262fa90] +[b05n11:176582] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n11:176582] [ 3] [a32n14:152748] *** Process received signal *** +[a32n14:152748] Signal: Segmentation fault (11) +[a32n14:152748] Signal code: (3) +[a32n14:152748] Failing at address: 0x14170000000000a0 +[a32n14:152748] [ 0] [0x1000000504d8] +[a32n14:152748] [ 1] [0x3fffde0d6c80] +[a32n14:152748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n14:152748] [ 3] [a32n07:172877] *** Process received signal *** +[a32n07:172877] Signal: Segmentation fault (11) +[a32n07:172877] Signal code: (3) +[a32n07:172877] Failing at address: 0xb170000000000a0 +[b10n15:164412] *** Process received signal *** +[b10n15:164412] Signal: Segmentation fault (11) +[b10n15:164412] Signal code: (3) +[b10n15:164412] Failing at address: 0xb170000000000a0 +[b10n15:164412] [ 0] [0x1000000504d8] +[b10n15:164412] [ 1] [0x3fffe0e05100] +[b10n15:164412] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n15:164412] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n15:164412] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n15:164412] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n15:164412] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n15:164412] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n15:164412] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n15:164412] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n15:164412] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n15:164412] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n15:164412] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n15:164412] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n15:164412] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n15:164412] *** End of error message *** +[a31n18:169359] *** Process received signal *** +[a31n18:169359] Signal: Segmentation fault (11) +[a31n18:169359] Signal code: (3) +[a31n18:169359] Failing at address: 0xb170000000000a0 +[a31n18:169359] [ 0] [a31n10:155397] *** Process received signal *** +[a31n10:155397] Signal: Segmentation fault (11) +[a31n10:155397] Signal code: (3) +[a31n10:155397] Failing at address: 0xb170000000000a0 +[a31n10:155397] [ 0] [0x1000000504d8] +[a31n10:155397] [ 1] [b13n02:125267] *** Process received signal *** +[b13n02:125267] Signal: Segmentation fault (11) +[b13n02:125267] Signal code: (3) +[b13n02:125267] Failing at address: 0x5170000000000a0 +[b13n02:125267] [ 0] [0x1000000504d8] +[b13n02:125267] [a31n14:165665] *** Process received signal *** +[a31n14:165665] Signal: Segmentation fault (11) +[a31n14:165665] Signal code: (3) +[a31n14:165665] Failing at address: 0x14170000000000a0 +[a31n14:165665] [ 0] [0x1000000504d8] +[a31n14:165665] [ 1] [b05n04:06836] *** Process received signal *** +[b05n04:06836] Signal: Segmentation fault (11) +[b05n04:06836] Signal code: (3) +[b05n04:06836] Failing at address: 0x5170000000000a0 +[b05n04:06836] [ 0] [0x1000000504d8] +[b05n04:06836] [ 1] [0x3fffd4684d70] +[b05n04:06836] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n04:06836] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n04:06836] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n04:06836] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n04:06836] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n04:06836] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n04:06836] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n04:06836] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n04:06836] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n04:06836] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n04:06836] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n04:06836] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n04:06836] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n04:06836] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n02:74429] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n02:74429] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n02:74429] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n02:74429] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n02:74429] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n02:74429] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n02:74429] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n02:74429] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n02:74429] *** End of error message *** +[a02n10:135852] *** Process received signal *** +[a02n10:135852] Signal: Segmentation fault (11) +[a02n10:135852] Signal code: (3) +[a02n10:135852] Failing at address: 0xb170000000000a0 +[a02n10:135852] [ 0] [0x1000000504d8] +[a02n10:135852] [ 1] [0x3fffda53d750] +[a02n10:135852] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n10:135852] [ 3] [b20n17:122356] *** Process received signal *** +[b20n17:122356] Signal: Segmentation fault (11) +[b20n17:122356] Signal code: (3) +[b20n17:122356] Failing at address: 0xb170000000000a0 +[b20n17:122356] [ 0] [0x1000000504d8] +[b20n17:122356] [ 1] [0x3ffff23c0510] +[b20n17:122356] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n17:122356] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n17:122356] [ 4] [a28n07:04683] *** Process received signal *** +[a28n07:04683] Signal: Segmentation fault (11) +[a28n07:04683] Signal code: (3) +[a28n07:04683] Failing at address: 0x14170000000000a0 +[a28n07:04683] [ 0] [0x1000000504d8] +[a28n07:04683] [ 1] [0x3fffed2ce1f0] +[a28n07:04683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n07:04683] [ 3] [a31n02:158419] *** Process received signal *** +[a31n02:158419] Signal: Segmentation fault (11) +[a31n02:158419] Signal code: (3) +[a31n02:158419] Failing at address: 0x14170000000000a0 +[a31n02:158419] [ 0] [0x1000000504d8] +[a31n02:158419] [ 1] [0x3ffffbe12f70] +[a31n02:158419] [a27n07:45305] *** Process received signal *** +[a27n07:45305] Signal: Segmentation fault (11) +[a27n07:45305] Signal code: (3) +[a27n07:45305] Failing at address: 0xb170000000000a0 +[a27n07:45305] [ 0] [0x1000000504d8] +[a27n07:45305] [ 1] [0x3fffe5e15e10] +[a27n07:45305] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n07:45305] [ 3] [b06n03:164498] *** Process received signal *** +[b06n03:164498] Signal: Segmentation fault (11) +[b06n03:164498] Signal code: (3) +[b06n03:164498] Failing at address: 0x14170000000000a0 +[b06n03:164498] [ 0] [0x1000000504d8] +[b06n03:164498] [ 1] [0x3ffff2f70e90] +[b06n03:164498] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n03:164498] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n03:164498] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n03:164498] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n03:164498] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n03:164498] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n03:164498] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n03:164498] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n03:164498] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n03:164498] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n03:164498] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n03:164498] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n03:164498] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n03:164498] *** End of error message *** +[b07n02:174057] *** Process received signal *** +[b07n02:174057] Signal: Segmentation fault (11) +[b07n02:174057] Signal code: (3) +[b07n02:174057] Failing at address: 0x14170000000000a0 +[b07n02:174057] [ 0] [0x1000000504d8] +[b07n02:174057] [ 1] [a31n13:155851] *** Process received signal *** +[a31n13:155851] Signal: Segmentation fault (11) +[a31n13:155851] Signal code: (3) +[a31n13:155851] Failing at address: 0x14170000000000a0 +[a31n13:155851] [ 0] [b13n04:129407] *** Process received signal *** +[b13n04:129407] Signal: Segmentation fault (11) +[b13n04:129407] Signal code: (3) +[b13n04:129407] Failing at address: 0xb170000000000a0 +[b13n04:129407] [ 0] [0x1000000504d8] +[b13n04:129407] [ 1] [a35n05:149465] *** Process received signal *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n16:124962] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n12:160189] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n12:160189] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n09:170657] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n09:170657] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n09:170657] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n09:170657] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n09:170657] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n10:122761] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n10:122761] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n14:126120] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n14:126120] [14] [0x3fffe48b2ed0] +[b20n11:123521] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n11:123521] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n11:123521] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n11:123521] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n11:123521] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n11:123521] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n11:123521] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n11:123521] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n11:123521] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n11:123521] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n17:163178] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n17:163178] [ 5] [b22n07:120698] *** Process received signal *** +[b22n07:120698] Signal: Segmentation fault (11) +[b22n07:120698] Signal code: (3) +[b22n07:120698] Failing at address: 0xb170000000000a0 +[b22n07:120698] [ 0] [0x1000000504d8] +[b22n07:120698] [ 1] [0x3ffff7c1ac00] +[b22n07:120698] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n07:120698] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n07:120698] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n07:120698] [ 5] [a34n02:139631] *** Process received signal *** +[a34n02:139631] Signal: Segmentation fault (11) +[a34n02:139631] Signal code: (3) +[a34n02:139631] Failing at address: 0x14170000000000a0 +[a34n02:139631] [ 0] [0x1000000504d8] +[a34n02:139631] [ 1] [0x3ffff6b512b0] +[a34n02:139631] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n02:139631] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n02:139631] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n07:02841] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n07:02841] [ 5] [0x1000000504d8] +[b17n03:135036] [ 1] [0x3fffd60cda10] +[b17n03:135036] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n03:135036] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n03:135036] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n03:135036] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n09:128494] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n09:128494] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n09:128494] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n09:128494] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n09:128494] [ 9] [b07n05:176910] *** Process received signal *** +[b07n05:176910] Signal: Segmentation fault (11) +[b07n05:176910] Signal code: (3) +[b07n05:176910] Failing at address: 0x14170000000000a0 +[b07n05:176910] [ 0] [0x1000000504d8] +[b07n05:176910] [ 1] [0x3ffff63818e0] +[b07n05:176910] [ 2] [b15n08:118553] *** Process received signal *** +[b15n08:118553] Signal: Segmentation fault (11) +[b15n08:118553] Signal code: (3) +[b15n08:118553] Failing at address: 0xb170000000000a0 +[b15n08:118553] [ 0] [0x1000000504d8] +[b15n08:118553] [ 1] [0x3ffff93bf420] +[b15n08:118553] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n08:118553] [ 3] [a02n08:146119] *** Process received signal *** +[a02n08:146119] Signal: Segmentation fault (11) +[a02n08:146119] Signal code: (3) +[a02n08:146119] Failing at address: 0xb170000000000a0 +[a02n08:146119] [ 0] [0x1000000504d8] +[a02n08:146119] [ 1] [0x3fffd86d5ca0] +[a02n08:146119] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n08:146119] [ 3] [b20n15:132615] *** Process received signal *** +[b20n15:132615] Signal: Segmentation fault (11) +[b20n15:132615] Signal code: (3) +[b20n15:132615] Failing at address: 0xb170000000000a0 +[b20n15:132615] [ 0] [0x1000000504d8] +[b20n15:132615] [ 1] [0x3fffdff34760] +[b20n15:132615] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n15:132615] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n15:132615] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n15:132615] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n15:132615] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n15:132615] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n15:132615] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n15:132615] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n15:132615] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n15:132615] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n15:132615] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n15:132615] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n15:132615] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n15:132615] *** End of error message *** +[a31n07:156486] *** Process received signal *** +[a31n07:156486] Signal: Segmentation fault (11) +[a31n07:156486] Signal code: (3) +[a31n07:156486] Failing at address: 0xb170000000000a0 +[a31n07:156486] [ 0] [0x1000000504d8] +[a31n07:156486] [ 1] [0x3fffec6c8d70] +[a31n07:156486] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n07:156486] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n07:156486] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n07:156486] [ 5] [a29n04:156413] *** Process received signal *** +[a29n04:156413] Signal: Segmentation fault (11) +[a29n04:156413] Signal code: (3) +[a29n04:156413] Failing at address: 0x14170000000000a0 +[a29n04:156413] [ 0] [0x1000000504d8] +[a29n04:156413] [ 1] [b13n03:126187] *** Process received signal *** +[b09n15:165474] *** Process received signal *** +[b09n15:165474] Signal: Segmentation fault (11) +[b09n15:165474] Signal code: (3) +[b09n15:165474] Failing at address: 0xb170000000000a0 +[b09n15:165474] [ 0] [0x1000000504d8] +[b09n15:165474] [ 1] [0x3fffff4804e0] +[b09n15:165474] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n15:165474] [ 3] [a34n13:137744] *** Process received signal *** +[a34n13:137744] Signal: Segmentation fault (11) +[a34n13:137744] Signal code: (3) +[a34n13:137744] Failing at address: 0xb170000000000a0 +[a34n13:137744] [ 0] [0x1000000504d8] +[a34n13:137744] [ 1] [0x3fffd9a7a630] +[a34n13:137744] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n03:68833] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n03:68833] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n03:68833] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n03:68833] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n03:68833] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n03:68833] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n03:68833] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n03:68833] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n03:68833] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n03:68833] *** End of error message *** +[a33n03:147537] *** Process received signal *** +[a33n03:147537] Signal: Segmentation fault (11) +[a33n03:147537] Signal code: (3) +[a33n03:147537] Failing at address: 0x14170000000000a0 +[a33n03:147537] [ 0] [0x1000000504d8] +[a33n03:147537] [ 1] [0x3fffc3fbb990] +[a33n03:147537] [ 2] [a32n08:165282] *** Process received signal *** +[a32n08:165282] Signal: Segmentation fault (11) +[a32n08:165282] Signal code: (3) +[a32n08:165282] Failing at address: 0xb170000000000a0 +[a32n08:165282] [ 0] [0x1000000504d8] +[a32n08:165282] [ 1] [0x3ffffd6e7be0] +[a32n08:165282] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n08:165282] [ 3] [b15n11:134748] *** Process received signal *** +[b15n11:134748] Signal: Segmentation fault (11) +[b15n11:134748] Signal code: (3) +[b15n11:134748] Failing at address: 0xb170000000000a0 +[b15n11:134748] [ 0] [0x1000000504d8] +[b15n11:134748] [ 1] [0x3fffefaa6a40] +[b15n11:134748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n11:134748] [ 3] [a35n12:145262] *** Process received signal *** +[a35n12:145262] Signal: Segmentation fault (11) +[a35n12:145262] Signal code: (3) +[a35n12:145262] Failing at address: 0xb170000000000a0 +[a35n12:145262] [ 0] [0x1000000504d8] +[a35n12:145262] [ 1] [0x3fffed3cc430] +[a35n12:145262] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n12:145262] [ 3] [b12n18:123866] *** Process received signal *** +[b12n18:123866] Signal: Segmentation fault (11) +[b12n18:123866] Signal code: (3) +[b12n18:123866] Failing at address: 0x14170000000000a0 +[b12n18:123866] [ 0] [0x1000000504d8] +[b12n18:123866] [ 1] [0x3fffe40a9650] +[b12n18:123866] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n18:123866] [ 3] [b06n10:176260] *** Process received signal *** +[b06n10:176260] Signal: Segmentation fault (11) +[b06n10:176260] Signal code: (3) +[b06n10:176260] Failing at address: 0x14170000000000a0 +[b06n10:176260] [ 0] [0x1000000504d8] +[b06n10:176260] [ 1] [0x3ffffc66d190] +[b06n10:176260] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n10:176260] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n10:176260] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n10:176260] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n08:124116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n08:124116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n08:124116] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n08:124116] [11] [b14n14:122444] *** Process received signal *** +[b14n14:122444] Signal: Segmentation fault (11) +[b14n14:122444] Signal code: (3) +[b14n14:122444] Failing at address: 0xb170000000000a0 +[b14n14:122444] [ 0] [0x1000000504d8] +[b14n14:122444] [ 1] [0x3fffc9248790] +[b14n14:122444] [ 2] [b07n07:163253] *** Process received signal *** +[b07n07:163253] Signal: Segmentation fault (11) +[b07n07:163253] Signal code: (3) +[b07n07:163253] Failing at address: 0x14170000000000a0 +[b07n07:163253] [ 0] [0x1000000504d8] +[b07n07:163253] [ 1] [0x3fffd901f530] +[b07n07:163253] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n07:163253] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n12:120401] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n12:120401] [ 5] [b09n05:173823] *** Process received signal *** +[b09n05:173823] Signal: Segmentation fault (11) +[b09n05:173823] Signal code: (3) +[b09n05:173823] Failing at address: 0xb170000000000a0 +[b09n05:173823] [ 0] [0x1000000504d8] +[b09n05:173823] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n16:129237] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n16:129237] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n04:136610] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n04:136610] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n04:136610] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n04:136610] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n04:136610] [ 8] [a29n13:165388] *** Process received signal *** +[a29n13:165388] Signal: Segmentation fault (11) +[a29n13:165388] Signal code: (3) +[a29n13:165388] Failing at address: 0xb170000000000a0 +[a29n13:165388] [ 0] [0x1000000504d8] +[a29n13:165388] [ 1] [0x3fffe7297e60] +[a29n13:165388] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n13:165388] [ 3] [a34n06:148310] [ 0] [0x1000000504d8] +[a34n06:148310] [ 1] [0x3fffda631270] +[a34n06:148310] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n06:148310] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n12:168644] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n12:168644] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n12:168644] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n12:168644] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n12:168644] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n12:168644] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n12:168644] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n12:168644] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n12:168644] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n12:168644] [13] [0x3fffcd700b30] +[b20n03:124680] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n03:124680] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n03:124680] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n07:116114] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n07:116114] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n07:116114] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n07:116114] [12] [a29n03:155728] *** Process received signal *** +[a29n03:155728] Signal: Segmentation fault (11) +[a29n03:155728] Signal code: (3) +[a29n03:155728] Failing at address: 0xb170000000000a0 +[a29n03:155728] [ 0] [0x1000000504d8] +[a29n03:155728] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n07:156144] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n07:156144] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n07:156144] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n07:156144] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n07:156144] *** End of error message *** +[b22n14:116411] *** Process received signal *** +[b22n14:116411] Signal: Segmentation fault (11) +[b22n14:116411] Signal code: (3) +[b22n14:116411] Failing at address: 0x14170000000000a0 +[b22n14:116411] [ 0] [0x1000000504d8] +[b22n14:116411] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n05:53376] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n05:53376] *** End of error message *** +[b08n04:172232] *** Process received signal *** +[b08n04:172232] Signal: Segmentation fault (11) +[b08n04:172232] Signal code: (3) +[b08n04:172232] Failing at address: 0x14170000000000a0 +[b08n04:172232] [ 0] [0x1000000504d8] +[b08n04:172232] [ 1] [0x3fffdb8380c0] +[b08n04:172232] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n04:172232] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n04:172232] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n04:172232] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n04:172232] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n04:172232] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n04:172232] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n04:172232] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n04:172232] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n04:172232] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n04:172232] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n04:172232] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n04:172232] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n04:172232] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n18:121303] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n18:121303] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n18:121303] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n17:08308] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n17:08308] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n09:166572] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n09:166572] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n09:166572] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n09:166572] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n09:166572] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n09:166572] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n09:166572] [13] [a28n12:177497] *** Process received signal *** +[a28n12:177497] Signal: Segmentation fault (11) +[a28n12:177497] Signal code: (3) +[a28n12:177497] Failing at address: 0x14170000000000a0 +[a28n12:177497] [ 0] [0x1000000504d8] +[a28n12:177497] [ 1] [0x3fffddbb2e00] +[a28n12:177497] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n12:177497] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n12:177497] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n12:177497] [ 5] [0x1000000504d8] +[b09n04:177599] [ 1] [0x3fffd7c97210] +[b09n04:177599] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n04:177599] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n04:177599] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n04:177599] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n05:05608] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n05:05608] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n05:05608] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n05:05608] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n05:05608] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n05:05608] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n05:05608] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n05:05608] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n05:05608] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n05:05608] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n05:05608] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n16:147288] [14] [b01n13:177638] *** Process received signal *** +[b01n13:177638] Signal: Segmentation fault (11) +[b01n13:177638] Signal code: (3) +[b01n13:177638] Failing at address: 0x5170000000000a0 +[b01n13:177638] [ 0] [0x1000000504d8] +[b01n13:177638] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n14:139845] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n14:139845] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n14:139845] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n04:117354] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n04:117354] [ 6] [a26n13:54241] *** Process received signal *** +[a26n13:54241] Signal: Segmentation fault (11) +[a26n13:54241] Signal code: (3) +[a26n13:54241] Failing at address: 0x14170000000000a0 +[a26n13:54241] [ 0] [0x1000000504d8] +[a26n13:54241] [ 1] [b20n05:121826] *** Process received signal *** +[b20n05:121826] Signal: Segmentation fault (11) +[b20n05:121826] Signal code: (3) +[b20n05:121826] Failing at address: 0x14170000000000a0 +[b20n05:121826] [ 0] [b22n01:122013] *** Process received signal *** +[b22n01:122013] Signal: Segmentation fault (11) +[b22n01:122013] Signal code: (3) +[b22n01:122013] Failing at address: 0x14170000000000a0 +[b22n01:122013] [ 0] [0x1000000504d8] +[b22n01:122013] [ 1] [0x3fffc8a220f0] +[b22n01:122013] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n01:122013] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n01:122013] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n01:122013] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n01:122013] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n01:122013] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n01:122013] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n01:122013] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n01:122013] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n01:122013] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n01:122013] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n01:122013] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n01:122013] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n01:122013] *** End of error message *** +[b02n04:168490] *** Process received signal *** +[b02n04:168490] Signal: Segmentation fault (11) +[b02n04:168490] Signal code: (3) +[b02n04:168490] Failing at address: 0x14170000000000a0 +[b02n04:168490] [ 0] [0x1000000504d8] +[b02n04:168490] [ 1] [0x3fffd1925db0] +[b02n04:168490] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n16:168050] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n16:168050] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n16:168050] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n16:168050] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n16:168050] [10] [a33n07:149477] *** Process received signal *** +[a33n07:149477] Signal: Segmentation fault (11) +[a33n07:149477] Signal code: (3) +[a33n07:149477] Failing at address: 0xb170000000000a0 +[a33n07:149477] [ 0] [0x1000000504d8] +[a33n07:149477] [ 1] [0x3fffe241a8e0] +[a33n07:149477] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n07:149477] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n07:149477] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n07:149477] [ 5] [b02n18:03311] *** Process received signal *** +[b02n18:03311] Signal: Segmentation fault (11) +[b02n18:03311] Signal code: (3) +[b02n18:03311] Failing at address: 0x14170000000000a0 +[b02n18:03311] [ 0] [0x1000000504d8] +[b02n18:03311] [ 1] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n13:131759] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n13:131759] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n13:131759] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n13:131759] *** End of error message *** +[0x1000000504d8] +[b13n09:143898] [ 1] [0x3fffedb59310] +[b13n09:143898] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n09:143898] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n09:143898] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n09:143898] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n08:144853] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n08:144853] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n08:144853] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n08:144853] [13] [0x1000000504d8] +[b20n12:119805] [ 1] [0x3fffd6767620] +[b20n12:119805] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n12:119805] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n12:119805] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n12:119805] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n13:122188] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n13:122188] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n14:120638] [ 4] [a35n08:138175] *** Process received signal *** +[a35n08:138175] Signal: Segmentation fault (11) +[a35n08:138175] Signal code: (3) +[a35n08:138175] Failing at address: 0xb170000000000a0 +[a35n08:138175] [ 0] [b19n18:122556] *** Process received signal *** +[b19n18:122556] Signal: Segmentation fault (11) +[b19n18:122556] Signal code: (3) +[b19n18:122556] Failing at address: 0x14170000000000a0 +[b19n18:122556] [ 0] [0x1000000504d8] +[b19n18:122556] [ 1] [0x3fffd0ece850] +[b19n18:122556] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n18:122556] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n18:122556] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n16:130375] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n16:130375] [ 5] [b10n10:126194] *** Process received signal *** +[b10n10:126194] Signal: Segmentation fault (11) +[b10n10:126194] Signal code: (3) +[b10n10:126194] Failing at address: 0xb170000000000a0 +[b10n10:126194] [ 0] [0x1000000504d8] +[b10n10:126194] [ 1] [0x3fffc8e78380] +[b10n10:126194] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n10:126194] [ 3] [b17n17:125809] [ 0] [0x1000000504d8] +[b17n17:125809] [ 1] [0x3ffffb1c92b0] +[b17n17:125809] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n17:125809] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n17:125809] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n09:124192] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n09:124192] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n09:124192] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n09:124192] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n09:124192] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n09:124192] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n09:124192] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n09:124192] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n09:124192] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n09:124192] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n07:174562] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n07:174562] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n07:174562] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n07:174562] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n07:174562] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n12:180116] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n12:180116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n12:180116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n12:180116] [10] [a36n15:151251] *** Process received signal *** +[a36n15:151251] Signal: Segmentation fault (11) +[a36n15:151251] Signal code: (3) +[a36n15:151251] Failing at address: 0x14170000000000a0 +[a36n15:151251] [ 0] [0x1000000504d8] +[a36n15:151251] [ 1] [0x3fffd9e25700] +[a36n15:151251] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n15:151251] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n15:32401] [ 9] [a35n13:138090] *** Process received signal *** +[a35n13:138090] Signal: Segmentation fault (11) +[a35n13:138090] Signal code: (3) +[a35n13:138090] Failing at address: 0x5170000000000a0 +[a35n13:138090] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n18:173978] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n18:173978] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n18:173978] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n18:173978] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n18:173978] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n18:173978] [10] [b22n15:120057] *** Process received signal *** +[b22n15:120057] Signal: Segmentation fault (11) +[b22n15:120057] Signal code: (3) +[b22n15:120057] Failing at address: 0x14170000000000a0 +[b22n15:120057] [ 0] [0x1000000504d8] +[b22n15:120057] [ 1] [0x3fffff1fb9a0] +[b22n15:120057] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n15:120057] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n15:120057] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n02:177547] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n02:177547] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n14:136137] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n14:136137] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n14:136137] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n14:136137] [ 8] [a34n10:141234] *** Process received signal *** +[a34n10:141234] Signal: Segmentation fault (11) +[a34n10:141234] Signal code: (3) +[a34n10:141234] Failing at address: 0x14170000000000a0 +[a34n10:141234] [ 0] [0x1000000504d8] +[a34n10:141234] [ 1] [0x3fffca22ce10] +[a34n10:141234] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n10:141234] [ 3] [b11n06:164014] *** Process received signal *** +[b11n06:164014] Signal: Segmentation fault (11) +[b11n06:164014] Signal code: (3) +[b11n06:164014] Failing at address: 0xb170000000000a0 +[b11n06:164014] [ 0] [0x1000000504d8] +[b11n06:164014] [ 1] [0x3fffe10e5a70] +[b11n06:164014] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n06:164014] [ 3] [0x3fffd32be140] +[b05n01:178523] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n01:178523] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n01:178523] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n01:178523] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n01:178523] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n01:163025] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n01:163025] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n01:163025] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n01:163025] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n01:163025] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n01:163025] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n01:163025] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n01:163025] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n01:163025] [14] [a35n15:148166] *** Process received signal *** +[a35n15:148166] Signal: Segmentation fault (11) +[a35n15:148166] Signal code: (3) +[a35n15:148166] Failing at address: 0x5170000000000a0 +[a35n15:148166] [ 0] [0x1000000504d8] +[a35n15:148166] [ 1] [0x3ffff1064bf0] +[a35n15:148166] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n15:148166] [ 3] [b04n18:176809] *** Process received signal *** +[b04n18:176809] Signal: Segmentation fault (11) +[b04n18:176809] Signal code: (3) +[b04n18:176809] Failing at address: 0xb170000000000a0 +[b04n18:176809] [ 0] [0x1000000504d8] +[b04n18:176809] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n07:08397] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n07:08397] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n07:08397] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n07:08397] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n07:08397] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n07:08397] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n07:08397] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n07:08397] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n07:173042] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n07:173042] *** End of error message *** +[b14n10:124920] *** Process received signal *** +[b14n10:124920] Signal: Segmentation fault (11) +[b14n10:124920] Signal code: (3) +[b14n10:124920] Failing at address: 0xb170000000000a0 +[b14n10:124920] [ 0] [0x1000000504d8] +[b14n10:124920] [ 1] [0x3ffff9804280] +[b14n10:124920] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n10:124920] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n10:124920] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n10:124920] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n10:124920] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n10:124920] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n10:124920] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n10:124920] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n10:124920] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n10:124920] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n10:124920] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n10:124920] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n10:124920] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n10:124920] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n15:119251] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n15:119251] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n15:119251] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n15:119251] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n11:124445] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n13:166926] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n13:166926] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n13:166926] [ 6] [a34n14:150134] *** Process received signal *** +[a34n14:150134] Signal: Segmentation fault (11) +[a34n14:150134] Signal code: (3) +[a34n14:150134] Failing at address: 0xb170000000000a0 +[a34n14:150134] [ 0] [0x1000000504d8] +[a34n14:150134] [ 1] [0x3fffdf3665a0] +[a34n14:150134] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n14:150134] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n14:150134] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n14:119483] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n12:164381] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n12:164381] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n12:164381] [ 9] [b08n05:173849] *** Process received signal *** +[b08n05:173849] Signal: Segmentation fault (11) +[b08n05:173849] Signal code: (3) +[b08n05:173849] Failing at address: 0xb170000000000a0 +[b08n05:173849] [ 0] [0x1000000504d8] +[b08n05:173849] [ 1] [0x3fffde4315d0] +[b08n05:173849] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n05:173849] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n13:117511] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n13:117511] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n13:117511] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n13:117511] [ 9] [a28n17:168160] *** Process received signal *** +[a28n17:168160] Signal: Segmentation fault (11) +[a28n17:168160] Signal code: (3) +[a28n17:168160] Failing at address: 0xb170000000000a0 +[a28n17:168160] [ 0] [0x1000000504d8] +[a28n17:168160] [ 1] [0x3fffd6ddb370] +[a28n17:168160] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n05:125614] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n05:125614] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n05:125614] [ 6] [0x3ffff9929d70] +[a29n11:162472] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n11:162472] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n11:162472] [ 4] [b22n02:119534] *** Process received signal *** +[b22n02:119534] Signal: Segmentation fault (11) +[b22n02:119534] Signal code: (3) +[b22n02:119534] Failing at address: 0x14170000000000a0 +[b22n02:119534] [ 0] [0x1000000504d8] +[b22n02:119534] [ 1] [0x3fffc744c3f0] +[b22n02:119534] [ 2] [0x1000000504d8] +[b23n10:126793] [ 1] [0x3fffd47d43d0] +[b23n10:126793] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n10:126793] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n10:126793] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n10:126793] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n10:126793] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n10:126793] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n10:126793] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n10:126793] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n10:126793] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n10:126793] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n10:126793] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n10:126793] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n10:126793] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n10:126793] *** End of error message *** +[0x1000000504d8] +[b09n18:164796] [ 1] [0x3fffd82e5ae0] +[b09n18:164796] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n18:164796] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n18:164796] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n04:126349] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n04:126349] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n04:126349] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n04:126349] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n04:126349] [11] [b09n12:178717] *** Process received signal *** +[b09n12:178717] Signal: Segmentation fault (11) +[b09n12:178717] Signal code: (3) +[b09n12:178717] Failing at address: 0x14170000000000a0 +[b09n12:178717] [ 0] [0x1000000504d8] +[b09n12:178717] [ 1] [0x3fffd3d96bc0] +[b09n12:178717] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n12:178717] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n16:173548] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n16:173548] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n16:173548] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n16:173548] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n16:173548] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n12:120131] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n12:120131] [ 5] [b13n17:129933] *** Process received signal *** +[b13n17:129933] Signal: Segmentation fault (11) +[b13n17:129933] Signal code: (3) +[b13n17:129933] Failing at address: 0xb170000000000a0 +[b13n17:129933] [ 0] [0x1000000504d8] +[b13n17:129933] [ 1] [0x3fffc1d77820] +[b13n17:129933] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n17:129933] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n17:129933] [ 4] [a33n16:149062] *** Process received signal *** +[a33n16:149062] Signal: Segmentation fault (11) +[a33n16:149062] Signal code: (3) +[a33n16:149062] Failing at address: 0x14170000000000a0 +[a33n16:149062] [ 0] [0x1000000504d8] +[a33n16:149062] [ 1] [0x3fffe5b35f10] +[a33n16:149062] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n16:149062] [ 3] [a32n04:153966] Signal: Segmentation fault (11) +[a32n04:153966] Signal code: (3) +[a32n04:153966] Failing at address: 0x5170000000000a0 +[a32n04:153966] [ 0] [0x1000000504d8] +[a32n04:153966] [ 1] [0x3fffe66e18c0] +[a32n04:153966] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n04:153966] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n06:135850] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n06:135850] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n06:135850] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n06:135850] [14] [a31n03:158656] *** Process received signal *** +[a31n03:158656] Signal: Segmentation fault (11) +[a31n03:158656] Signal code: (3) +[a31n03:158656] Failing at address: 0xb170000000000a0 +[a31n03:158656] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n11:153420] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n11:153420] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n11:153420] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n11:153420] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n11:153420] *** End of error message *** +[b01n10:173681] *** Process received signal *** +[b01n10:173681] Signal: Segmentation fault (11) +[b01n10:173681] Signal code: (3) +[b01n10:173681] Failing at address: 0x14170000000000a0 +[b01n10:173681] [ 0] [0x1000000504d8] +[b01n10:173681] [ 1] [0x3fffc91fd370] +[b01n10:173681] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n10:173681] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n10:173681] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n18:177222] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n18:177222] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n18:177222] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n18:177222] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n18:177222] [14] [a35n09:137775] *** Process received signal *** +[a35n09:137775] Signal: Segmentation fault (11) +[a35n09:137775] Signal code: (3) +[a35n09:137775] Failing at address: 0x14170000000000a0 +[a35n09:137775] [ 0] [0x1000000504d8] +[a35n09:137775] [ 1] [0x3fffd8bdf730] +[a35n09:137775] [ 2] [b04n04:179174] *** Process received signal *** +[b04n04:179174] Signal: Segmentation fault (11) +[b04n04:179174] Signal code: (3) +[b04n04:179174] Failing at address: 0x14170000000000a0 +[b04n04:179174] [ 0] [0x1000000504d8] +[b04n04:179174] [ 1] [0x3fffd33cd5c0] +[b04n04:179174] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n04:179174] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n04:179174] [ 4] [b13n10:134732] [ 0] [0x1000000504d8] +[b13n10:134732] [ 1] [0x3fffdcfac600] +[b13n10:134732] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n10:134732] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n10:134732] [ 4] [b15n18:08428] *** Process received signal *** +[b15n18:08428] Signal: Segmentation fault (11) +[b15n18:08428] Signal code: (3) +[b15n18:08428] Failing at address: 0xb170000000000a0 +[b15n18:08428] [ 0] [0x1000000504d8] +[b15n18:08428] [ 1] [0x3fffd0729aa0] +[b15n18:08428] [ 2] [a30n11:163259] *** Process received signal *** +[a30n11:163259] Signal: Segmentation fault (11) +[a30n11:163259] Signal code: (3) +[a30n11:163259] Failing at address: 0xb170000000000a0 +[a30n11:163259] [ 0] [0x1000000504d8] +[a30n11:163259] [ 1] [0x3fffd10e5ae0] +[a30n11:163259] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n11:163259] [ 3] [0x1000000504d8] +[a33n13:168161] [ 1] [0x3fffd1d1a3f0] +[a33n13:168161] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n13:168161] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n13:168161] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n13:168161] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n11:124434] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n11:124434] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n11:124434] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n11:124434] [ 9] [b11n09:161231] *** Process received signal *** +[b11n09:161231] Signal: Segmentation fault (11) +[b11n09:161231] Signal code: (3) +[b11n09:161231] Failing at address: 0x14170000000000a0 +[b11n09:161231] [ 0] [0x1000000504d8] +[b11n09:161231] [ 1] [0x3ffffdd04120] +[b11n09:161231] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n09:161231] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n13:123888] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n13:123888] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n13:123888] [ 6] [b19n15:124803] *** Process received signal *** +[b19n15:124803] Signal: Segmentation fault (11) +[b19n15:124803] Signal code: (3) +[b19n15:124803] Failing at address: 0x14170000000000a0 +[b19n15:124803] [ 0] [0x1000000504d8] +[b19n15:124803] [ 1] [0x3fffcb447fd0] +[b19n15:124803] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n15:124803] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n15:124803] [ 4] [a34n18:140384] *** Process received signal *** +[a34n18:140384] Signal: Segmentation fault (11) +[a34n18:140384] Signal code: (3) +[a34n18:140384] Failing at address: 0xb170000000000a0 +[a34n18:140384] [ 0] [0x1000000504d8] +[a34n18:140384] [ 1] [0x3fffe5f9d030] +[a34n18:140384] [ 2] [b02n05:164943] *** Process received signal *** +[b02n05:164943] Signal: Segmentation fault (11) +[b02n05:164943] Signal code: (3) +[b02n05:164943] Failing at address: 0xb170000000000a0 +[b02n05:164943] [ 0] [0x1000000504d8] +[b02n05:164943] [ 1] [0x3fffce7cb3d0] +[b02n05:164943] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n05:164943] [ 3] [a27n08:54626] *** Process received signal *** +[a27n08:54626] Signal: Segmentation fault (11) +[a27n08:54626] Signal code: (3) +[a27n08:54626] Failing at address: 0x14170000000000a0 +[a27n08:54626] [ 0] [0x1000000504d8] +[a27n08:54626] [ 1] [b14n18:131584] *** Process received signal *** +[b14n18:131584] Signal: Segmentation fault (11) +[b14n18:131584] Signal code: (3) +[b14n18:131584] Failing at address: 0xb170000000000a0 +[b14n18:131584] [ 0] [a32n10:162610] *** Process received signal *** +[a32n10:162610] Signal: Segmentation fault (11) +[a32n10:162610] Signal code: (3) +[a32n10:162610] Failing at address: 0x14170000000000a0 +[a32n10:162610] [ 0] [0x1000000504d8] +[a32n10:162610] [ 1] [b08n17:171198] *** Process received signal *** +[b08n17:171198] Signal: Segmentation fault (11) +[b08n17:171198] Signal code: (3) +[b08n17:171198] Failing at address: 0x5170000000000a0 +[b08n17:171198] [b19n14:133062] *** Process received signal *** +[b19n14:133062] Signal: Segmentation fault (11) +[b19n14:133062] Signal code: (3) +[b19n14:133062] Failing at address: 0xb170000000000a0 +[b19n14:133062] [ 0] [0x1000000504d8] +[b19n14:133062] [ 1] [0x3fffcfc35e10] +[b19n14:133062] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n14:133062] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n14:133062] [ 4] [b11n04:166208] *** Process received signal *** +[b11n04:166208] Signal: Segmentation fault (11) +[b11n04:166208] Signal code: (3) +[b11n04:166208] Failing at address: 0xb170000000000a0 +[b11n04:166208] [ 0] [0x1000000504d8] +[b11n04:166208] [ 1] [0x3fffdd7a02a0] +[b11n04:166208] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n04:166208] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n04:166208] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n08:116171] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n08:116171] [ 6] [b06n16:165920] *** Process received signal *** +[b06n16:165920] Signal: Segmentation fault (11) +[b06n16:165920] Signal code: (3) +[b06n16:165920] Failing at address: 0x14170000000000a0 +[b06n16:165920] [ 0] [0x1000000504d8] +[b06n16:165920] [ 1] [0x3ffff48ce4b0] +[b06n16:165920] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n16:165920] [ 3] [a27n04:52706] *** Process received signal *** +[a27n04:52706] Signal: Segmentation fault (11) +[a27n04:52706] Signal code: (3) +[a27n04:52706] Failing at address: 0x14170000000000a0 +[a27n04:52706] [ 0] [b21n07:117359] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n18:138475] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n18:138475] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n02:176049] *** End of error message *** +[b13n07:133372] *** Process received signal *** +[b13n07:133372] Signal: Segmentation fault (11) +[b13n07:133372] Signal code: (3) +[b13n07:133372] Failing at address: 0x14170000000000a0 +[b13n07:133372] [ 0] [0x1000000504d8] +[b13n07:133372] [ 1] [0x3fffd38554e0] +[b13n07:133372] [ 2] [a02n04:41670] *** Process received signal *** +[a02n04:41670] Signal: Segmentation fault (11) +[a02n04:41670] Signal code: (3) +[a02n04:41670] Failing at address: 0xb170000000000a0 +[a02n04:41670] [b17n16:127352] *** Process received signal *** +[b17n16:127352] Signal: Segmentation fault (11) +[b17n16:127352] Signal code: (3) +[b17n16:127352] Failing at address: 0xb170000000000a0 +[b17n16:127352] [ 0] [0x1000000504d8] +[b17n16:127352] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n18:07423] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n18:07423] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n18:07423] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n18:07423] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n18:07423] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n18:07423] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n18:07423] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n18:07423] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n18:07423] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n18:07423] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n18:07423] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n18:134183] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n18:134183] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n18:134183] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n18:134183] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n18:134183] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n18:134183] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n18:134183] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n18:134183] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n18:134183] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n18:134183] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n18:134183] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n18:134183] *** End of error message *** +[b12n09:128000] [ 0] [0x1000000504d8] +[b12n09:128000] [ 1] [0x3ffff33725c0] +[b12n09:128000] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n09:128000] [ 3] [a27n11:23062] *** Process received signal *** +[a27n11:23062] Signal: Segmentation fault (11) +[a27n11:23062] Signal code: (3) +[a27n11:23062] Failing at address: 0xb170000000000a0 +[a27n11:23062] [ 0] [0x1000000504d8] +[a27n11:23062] [ 1] [0x3fffc9b4b990] +[a27n11:23062] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n11:23062] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n13:122033] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n13:122033] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n16:128443] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n16:128443] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n16:128443] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n16:128443] [13] [b20n01:128829] *** Process received signal *** +[b20n01:128829] Signal: Segmentation fault (11) +[b20n01:128829] Signal code: (3) +[b20n01:128829] Failing at address: 0xb170000000000a0 +[b20n01:128829] [ 0] [a26n08:76223] *** Process received signal *** +[a26n08:76223] Signal: Segmentation fault (11) +[a26n08:76223] Signal code: (3) +[a26n08:76223] Failing at address: 0x5170000000000a0 +[a26n08:76223] [ 0] [a31n06:167367] *** Process received signal *** +[a31n06:167367] Signal: Segmentation fault (11) +[a31n06:167367] Signal code: (3) +[a31n06:167367] Failing at address: 0xb170000000000a0 +[a31n06:167367] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n18:69156] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n18:69156] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n18:69156] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n18:69156] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n18:69156] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n18:69156] [10] [b15n12:132099] Signal: Segmentation fault (11) +[b15n12:132099] Signal code: (3) +[b15n12:132099] Failing at address: 0x14170000000000a0 +[b15n12:132099] [ 0] [0x1000000504d8] +[b15n12:132099] [ 1] [0x3fffca47aea0] +[b15n12:132099] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n10:124501] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n10:124501] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n10:124501] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n10:124501] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n10:124501] *** End of error message *** +[a02n03:38504] *** Process received signal *** +[a02n03:38504] Signal: Segmentation fault (11) +[a02n03:38504] Signal code: (3) +[a02n03:38504] Failing at address: 0x14170000000000a0 +[a02n03:38504] [ 0] [0x1000000504d8] +[a02n03:38504] [ 1] [0x3fffc32bb7b0] +[a02n03:38504] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n03:38504] [ 3] [a02n18:21771] *** Process received signal *** +[a02n18:21771] Signal: Segmentation fault (11) +[a02n18:21771] Signal code: (3) +[a02n18:21771] Failing at address: 0x14170000000000a0 +[a02n18:21771] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n17:124166] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n17:124166] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n17:124166] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n17:124166] [ 7] [b07n08:172899] *** Process received signal *** +[b07n08:172899] Signal: Segmentation fault (11) +[b07n08:172899] Signal code: (3) +[b07n08:172899] Failing at address: 0xb170000000000a0 +[b07n08:172899] [ 0] [0x1000000504d8] +[b07n08:172899] [ 1] [0x3fffd8fda6b0] +[b07n08:172899] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n08:172899] [ 3] [a34n09:149100] *** Process received signal *** +[a34n09:149100] Signal: Segmentation fault (11) +[a34n09:149100] Signal code: (3) +[a34n09:149100] Failing at address: 0xb170000000000a0 +[a34n09:149100] [ 0] [0x1000000504d8] +[a34n09:149100] [ 1] [0x3fffdc363b50] +[a34n09:149100] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n09:149100] [ 3] [b07n11:170897] *** Process received signal *** +[b07n11:170897] Signal: Segmentation fault (11) +[b07n11:170897] Signal code: (3) +[b07n11:170897] Failing at address: 0x14170000000000a0 +[b07n11:170897] [ 0] [0x1000000504d8] +[b07n11:170897] [ 1] [0x3fffed092660] +[b07n11:170897] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n11:170897] [ 3] [b07n01:166960] *** Process received signal *** +[b07n01:166960] Signal: Segmentation fault (11) +[b07n01:166960] Signal code: (3) +[b07n01:166960] Failing at address: 0xb170000000000a0 +[b07n01:166960] [ 0] [0x1000000504d8] +[b07n01:166960] [ 1] [0x3ffff26a9470] +[b07n01:166960] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n01:166960] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n02:167597] *** End of error message *** +[b10n14:168374] *** Process received signal *** +[b10n14:168374] Signal: Segmentation fault (11) +[b10n14:168374] Signal code: (3) +[b10n14:168374] Failing at address: 0x14170000000000a0 +[b10n14:168374] [ 0] [0x1000000504d8] +[b10n14:168374] [ 1] [0x3fffdcd143e0] +[b10n14:168374] [ 2] [b19n11:133973] *** Process received signal *** +[b19n11:133973] Signal: Segmentation fault (11) +[b19n11:133973] Signal code: (3) +[b19n11:133973] Failing at address: 0x14170000000000a0 +[b19n11:133973] [ 0] [0x1000000504d8] +[b19n11:133973] [ 1] [0x3fffd58e5a40] +[b19n11:133973] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n11:133973] [ 3] [b02n07:180194] *** Process received signal *** +[b02n07:180194] Signal: Segmentation fault (11) +[b02n07:180194] Signal code: (3) +[b02n07:180194] Failing at address: 0x14170000000000a0 +[b02n07:180194] [ 0] [0x1000000504d8] +[b02n07:180194] [ 1] [0x3fffe7183690] +[b02n07:180194] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n07:180194] [ 3] [a27n09:51331] *** Process received signal *** +[a27n09:51331] Signal: Segmentation fault (11) +[a27n09:51331] Signal code: (3) +[a27n09:51331] Failing at address: 0x5170000000000a0 +[a27n09:51331] [b07n14:176069] *** Process received signal *** +[b07n14:176069] Signal: Segmentation fault (11) +[b07n14:176069] Signal code: (3) +[b07n14:176069] Failing at address: 0x14170000000000a0 +[b07n14:176069] [ 0] [0x1000000504d8] +[b07n14:176069] [ 1] [0x3ffffc184070] +[b07n14:176069] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n14:176069] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n03:123619] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n03:123619] [10] [0x3fffec08f6b0] +[a31n05:158574] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n05:158574] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n05:158574] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n05:158574] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n05:158574] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n05:158574] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n05:158574] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n05:158574] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n05:158574] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n05:158574] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n05:158574] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n05:158574] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n05:158574] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n05:158574] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n13:114397] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n13:114397] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n13:114397] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n13:114397] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n14:08952] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n14:08952] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n14:08952] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n14:08952] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n14:08952] *** End of error message *** +[0x1000000504d8] +[a27n01:48806] [ 1] [0x3ffff8fef3b0] +[a27n01:48806] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n01:48806] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n01:48806] [ 4] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n12:12761] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n12:12761] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n12:12761] [ 8] [b19n06:126374] *** Process received signal *** +[b19n06:126374] Signal: Segmentation fault (11) +[b19n06:126374] Signal code: (3) +[b19n06:126374] Failing at address: 0x14170000000000a0 +[b19n06:126374] [ 0] [0x1000000504d8] +[b19n06:126374] [ 1] [0x3fffd5994450] +[b19n06:126374] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n06:126374] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n15:177059] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n15:177059] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n15:177059] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n15:177059] [10] [b10n11:160347] *** Process received signal *** +[b10n11:160347] Signal: Segmentation fault (11) +[b10n11:160347] Signal code: (3) +[b10n11:160347] Failing at address: 0x14170000000000a0 +[b10n11:160347] [ 0] [a31n15:155432] *** Process received signal *** +[a31n15:155432] Signal: Segmentation fault (11) +[a31n15:155432] Signal code: (3) +[a31n15:155432] Failing at address: 0x14170000000000a0 +[a31n15:155432] [ 0] [0x1000000504d8] +[a31n15:155432] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n10:142930] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n10:142930] [ 5] [0x3fffc254a230] +[b01n17:179116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n17:179116] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n17:179116] [ 4] [a34n16:135791] *** Process received signal *** +[a34n16:135791] Signal: Segmentation fault (11) +[a34n16:135791] Signal code: (3) +[a34n16:135791] Failing at address: 0x14170000000000a0 +[a34n16:135791] [ 0] [0x1000000504d8] +[a34n16:135791] [ 1] [0x3fffc235f5e0] +[a34n16:135791] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n16:135791] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n16:135791] [ 4] [b22n08:118581] *** Process received signal *** +[b22n08:118581] Signal: Segmentation fault (11) +[b22n08:118581] Signal code: (3) +[b22n08:118581] Failing at address: 0x14170000000000a0 +[b22n08:118581] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n17:149904] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n16:116525] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n16:116525] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n16:116525] [ 8] [b13n05:135166] *** Process received signal *** +[b13n05:135166] Signal: Segmentation fault (11) +[b13n05:135166] Signal code: (3) +[b13n05:135166] Failing at address: 0xb170000000000a0 +[b13n05:135166] [ 0] [0x1000000504d8] +[b13n05:135166] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n16:157888] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n16:157888] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n16:157888] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n16:157888] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n07:118166] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n07:118166] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n07:118166] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n09:123420] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n14:128870] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n14:128870] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n17:03846] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n17:03846] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n17:03846] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n17:03846] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n17:03846] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n17:03846] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n17:03846] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n17:03846] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n17:03846] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n17:03846] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n17:03846] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n02:06561] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n02:06561] [ 6] [a26n11:62422] *** Process received signal *** +[a26n11:62422] Signal: Segmentation fault (11) +[a26n11:62422] Signal code: (3) +[a26n11:62422] Failing at address: 0xb170000000000a0 +[a26n11:62422] [ 0] [0x1000000504d8] +[a26n11:62422] [ 1] [0x3fffc8c61870] +[a26n11:62422] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n11:62422] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n11:62422] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n11:62422] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n11:62422] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n11:62422] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n11:62422] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n11:62422] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n11:62422] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n11:62422] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n11:62422] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n11:62422] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n11:62422] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n11:62422] *** End of error message *** +[b02n01:167297] *** Process received signal *** +[b02n01:167297] Signal: Segmentation fault (11) +[b02n01:167297] Signal code: (3) +[b02n01:167297] Failing at address: 0xb170000000000a0 +[b02n01:167297] [ 0] [0x1000000504d8] +[b02n01:167297] [ 1] [b14n13:122144] *** Process received signal *** +[b14n13:122144] Signal: Segmentation fault (11) +[b14n13:122144] Signal code: (3) +[b14n13:122144] Failing at address: 0xb170000000000a0 +[b14n13:122144] [ 0] [0x3ffffb243c80] +[b01n18:179700] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n18:179700] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n18:179700] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n18:179700] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n18:179700] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n18:179700] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n09:03928] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n14:169522] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n14:133632] [ 5] [a28n13:176366] *** Process received signal *** +[a28n13:176366] Signal: Segmentation fault (11) +[a28n13:176366] Signal code: (3) +[a28n13:176366] Failing at address: 0x14170000000000a0 +[a28n13:176366] [ 0] [0x1000000504d8] +[a28n13:176366] [ 1] [0x3fffd6ab8fd0] +[a28n13:176366] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n13:176366] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n10:176311] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n10:176311] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n10:176311] [14] [ 1] [0x3ffff7a4d590] +[b22n12:122963] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n12:122963] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n12:122963] [ 4] [b02n06:43019] *** Process received signal *** +[b02n06:43019] Signal: Segmentation fault (11) +[b02n06:43019] Signal code: (3) +[b02n06:43019] Failing at address: 0x14170000000000a0 +[b02n06:43019] [ 0] [0x1000000504d8] +[b02n06:43019] [ 1] [a27n17:15874] *** Process received signal *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n01:141816] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n01:141816] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n01:141816] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n06:01838] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n06:01838] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n06:01838] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n06:01838] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n06:01838] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n06:01838] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n06:01838] [11] [b12n17:138141] *** Process received signal *** +[b12n17:138141] Signal: Segmentation fault (11) +[b12n17:138141] Signal code: (3) +[b12n17:138141] Failing at address: 0x14170000000000a0 +[b12n17:138141] [ 0] [0x1000000504d8] +[b12n17:138141] [ 1] [0x3fffd0159770] +[b12n17:138141] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n03:177925] *** Process received signal *** +[b05n03:177925] Signal: Segmentation fault (11) +[b05n03:177925] Signal code: (3) +[b05n03:177925] Failing at address: 0x14170000000000a0 +[b05n03:177925] [ 0] [0x1000000504d8] +[b05n03:177925] [ 1] [0x3fffeac1dc20] +[b05n03:177925] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n03:177925] [ 3] [b01n14:175714] *** Process received signal *** +[b01n14:175714] Signal: Segmentation fault (11) +[b01n14:175714] Signal code: (3) +[b01n14:175714] Failing at address: 0x14170000000000a0 +[b01n14:175714] [ 0] [0x1000000504d8] +[b01n14:175714] [ 1] [0x3fffc9e80a00] +[b01n14:175714] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n14:175714] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n14:175714] [ 4] [b02n02:166746] *** Process received signal *** +[b02n02:166746] Signal: Segmentation fault (11) +[b02n02:166746] Signal code: (3) +[b02n02:166746] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n14:159239] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n14:159239] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n14:159239] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n14:159239] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n06:143012] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n06:143012] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n06:143012] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n06:143012] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n06:143012] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n08:08957] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n08:08957] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n08:08957] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n08:08957] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n02:163030] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n02:163030] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n02:163030] *** End of error message *** +[a35n07:138202] *** Process received signal *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n11:171963] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n11:171963] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n11:171963] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n11:171963] *** End of error message *** +[a33n08:147823] *** Process received signal *** +[a33n08:147823] Signal: Segmentation fault (11) +[a33n08:147823] Signal code: (3) +[a33n08:147823] Failing at address: 0x14170000000000a0 +[a33n08:147823] [ 0] [0x1000000504d8] +[a33n08:147823] [ 1] [0x3fffdb58db30] +[a33n08:147823] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n08:147823] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n07:132089] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n07:132089] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n07:132089] [ 8] [b17n18:126904] *** Process received signal *** +[b17n18:126904] Signal: Segmentation fault (11) +[b17n18:126904] Signal code: (3) +[b17n18:126904] Failing at address: 0x14170000000000a0 +[a35n14:136661] *** Process received signal *** +[a35n14:136661] Signal: Segmentation fault (11) +[a35n14:136661] Signal code: (3) +[a35n14:136661] Failing at address: 0x5170000000000a0 +[a35n14:136661] [ 0] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n10:67857] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n10:67857] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n10:67857] [ 5] [b10n02:164488] *** Process received signal *** +[b10n02:164488] Signal: Segmentation fault (11) +[b10n02:164488] Signal code: (3) +[b10n02:164488] Failing at address: 0x14170000000000a0 +[b10n02:164488] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n07:153178] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n07:153178] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n04:169911] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n04:169911] *** End of error message *** +[b19n07:131671] *** Process received signal *** +[b19n07:131671] Signal: Segmentation fault (11) +[b19n07:131671] Signal code: (3) +[b19n07:131671] Failing at address: 0xb170000000000a0 +[b19n07:131671] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n16:124664] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n16:124664] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n16:124664] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n16:124664] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n09:121755] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n09:121755] [ 5] [b15n13:123709] *** Process received signal *** +[b15n13:123709] Signal: Segmentation fault (11) +[b15n13:123709] Signal code: (3) +[b15n13:123709] Failing at address: 0xb170000000000a0 +[b15n13:123709] [ 0] [0x1000000504d8] +[b15n13:123709] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n16:01573] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n16:01573] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n16:01573] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n11:152986] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n11:152986] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n13:156921] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n13:156921] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n13:156921] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n13:156921] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n13:156921] [10] [b12n03:139011] *** Process received signal *** +[b12n03:139011] Signal: Segmentation fault (11) +[b12n03:139011] Signal code: (3) +[b12n03:139011] Failing at address: 0x14170000000000a0 +[b12n03:139011] [ 0] [0x1000000504d8] +[b12n03:139011] [ 1] [0x3fffdc7bbdd0] +[b12n03:139011] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n03:139011] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n05:138196] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n05:138196] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n05:138196] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n05:138196] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n05:138196] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n05:138196] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n05:138196] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n05:138196] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n05:138196] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n05:138196] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n15:85940] *** End of error message *** +[0x3fffeb937d60] +[a28n03:10292] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n03:10292] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n03:10292] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n03:10292] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n03:10292] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n18:28221] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n18:28221] *** End of error message *** +[b11n17:164294] *** Process received signal *** +[b11n17:164294] Signal: Segmentation fault (11) +[b11n17:164294] Signal code: (3) +[b11n17:164294] Failing at address: 0xb170000000000a0 +[b11n17:164294] [ 0] [0x1000000504d8] +[b11n17:164294] [ 1] [0x3fffd5aedc80] +[b11n17:164294] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n17:164294] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n17:164294] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n17:164294] [ 5] [b18n01:128157] *** Process received signal *** +[b18n01:128157] Signal: Segmentation fault (11) +[b18n01:128157] Signal code: (3) +[b18n01:128157] Failing at address: 0x14170000000000a0 +[b18n01:128157] [ 0] [0x1000000504d8] +[b18n01:128157] [ 1] [0x3ffff847e360] +[b18n01:128157] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n01:128157] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n01:128157] [ 4] [a31n17:156181] *** Process received signal *** +[a31n17:156181] Signal: Segmentation fault (11) +[a31n17:156181] Signal code: (3) +[a31n17:156181] Failing at address: 0xb170000000000a0 +[a31n17:156181] [ 0] [0x1000000504d8] +[a31n17:156181] [ 1] [0x3ffff53973f0] +[a31n17:156181] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n17:156181] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n17:156181] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n11:01170] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n11:01170] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n08:124725] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n08:124725] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n08:124725] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n08:124725] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n08:124725] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n10:130582] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n10:130582] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n10:130582] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n10:130582] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n12:11141] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n12:11141] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n12:11141] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n12:11141] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n16:171918] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n16:171918] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n16:171918] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n16:171918] [ 9] [b12n02:143967] *** Process received signal *** +[b12n02:143967] Signal: Segmentation fault (11) +[b12n02:143967] Signal code: (3) +[b12n02:143967] Failing at address: 0xb170000000000a0 +[b12n02:143967] [ 0] [0x1000000504d8] +[b12n02:143967] [ 1] [0x3fffc046a5b0] +[b12n02:143967] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n02:143967] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n09:119548] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n09:119548] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n09:119548] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n09:173750] *** End of error message *** +[b21n06:122164] *** Process received signal *** +[b21n06:122164] Signal: Segmentation fault (11) +[b21n06:122164] Signal code: (3) +[b21n06:122164] Failing at address: 0xb170000000000a0 +[b21n06:122164] [ 0] [0x1000000504d8] +[b21n06:122164] [ 1] [0x3fffeba27930] +[b21n06:122164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n06:122164] [ 3] [a36n17:150927] *** Process received signal *** +[a36n17:150927] Signal: Segmentation fault (11) +[a36n17:150927] Signal code: (3) +[a36n17:150927] Failing at address: 0x5170000000000a0 +[a36n17:150927] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n16:153290] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n16:153290] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n16:153290] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n16:153290] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n16:153290] [14] [a36n11:152842] *** Process received signal *** +[a36n11:152842] Signal: Segmentation fault (11) +[a36n11:152842] Signal code: (3) +[a36n11:152842] Failing at address: 0xb170000000000a0 +[a36n11:152842] [ 0] [0x1000000504d8] +[a36n11:152842] [ 1] [0x3fffe10a0360] +[a36n11:152842] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n11:152842] [ 3] [b12n11:137872] *** Process received signal *** +[b12n11:137872] Signal: Segmentation fault (11) +[b12n11:137872] Signal code: (3) +[b12n11:137872] Failing at address: 0x5170000000000a0 +[b12n11:137872] [ 0] [0x1000000504d8] +[b12n11:137872] [ 1] [0x3fffdd5beec0] +[b12n11:137872] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n11:137872] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n11:137872] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n11:137872] [ 5] [b11n14:161618] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n15:176425] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n15:176425] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n15:176425] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n15:176425] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n15:176425] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n15:176425] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n15:176425] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n15:176425] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n15:176425] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n15:176425] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n15:176425] *** End of error message *** +[0x1000000504d8] +[b03n13:01300] [ 1] [0x3ffffebc41d0] +[b03n13:01300] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n13:01300] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n13:01300] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n09:123815] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n08:174411] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n08:174411] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n08:174411] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n08:174411] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n08:174411] [13] [b21n01:130410] *** Process received signal *** +[b21n01:130410] Signal: Segmentation fault (11) +[b21n01:130410] Signal code: (3) +[b21n01:130410] Failing at address: 0xb170000000000a0 +[b21n01:130410] [ 0] [0x1000000504d8] +[b21n01:130410] [ 1] [0x3fffe6aa3510] +[b21n01:130410] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n01:130410] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n01:130410] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n02:162035] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n02:162035] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n02:162035] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n02:162035] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n02:162035] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n02:162035] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n02:162035] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n02:162035] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n02:162035] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n02:162035] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n02:162035] *** End of error message *** +[b04n15:177860] *** Process received signal *** +[b04n15:177860] Signal: Segmentation fault (11) +[b04n15:177860] Signal code: (3) +[b04n15:177860] Failing at address: 0x5170000000000a0 +[b04n15:177860] [ 0] [0x1000000504d8] +[b04n15:177860] [ 1] [0x3fffc47bca50] +[b04n15:177860] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n15:177860] [ 3] [b14n16:122495] *** Process received signal *** +[b14n16:122495] Signal: Segmentation fault (11) +[b14n16:122495] Signal code: (3) +[b14n16:122495] Failing at address: 0x14170000000000a0 +[b14n16:122495] [ 0] [0x1000000504d8] +[b14n16:122495] [ 1] [0x3ffffdc93c50] +[b14n16:122495] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n16:122495] [ 3] [0x3fffc2d73730] +[a31n01:173778] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n01:173778] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n01:173778] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n01:173778] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n01:173778] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n01:173778] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n01:173778] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n12:123233] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n12:123233] [ 5] [b21n03:127630] *** Process received signal *** +[b21n03:127630] Signal: Segmentation fault (11) +[b21n03:127630] Signal code: (3) +[b21n03:127630] Failing at address: 0x14170000000000a0 +[b21n03:127630] [ 0] [0x1000000504d8] +[b21n03:127630] [ 1] [0x3ffffc211340] +[b21n03:127630] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n03:127630] [ 3] [b10n17:164560] *** Process received signal *** +[b10n17:164560] Signal: Segmentation fault (11) +[b10n17:164560] Signal code: (3) +[b10n17:164560] Failing at address: 0xb170000000000a0 +[b10n17:164560] [ 0] [0x1000000504d8] +[b10n17:164560] [ 1] [0x3fffe12e2450] +[b10n17:164560] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n17:164560] [ 3] [b22n17:119582] *** Process received signal *** +[b22n17:119582] Signal: Segmentation fault (11) +[b22n17:119582] Signal code: (3) +[b22n17:119582] Failing at address: 0xb170000000000a0 +[b22n17:119582] [ 0] [0x1000000504d8] +[b22n17:119582] [ 1] [0x3fffd9337b70] +[b22n17:119582] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n17:119582] [ 3] [b12n01:145734] *** Process received signal *** +[b12n01:145734] Signal: Segmentation fault (11) +[b12n01:145734] Signal code: (3) +[b12n01:145734] Failing at address: 0x14170000000000a0 +[b12n01:145734] [ 0] [0x1000000504d8] +[b12n01:145734] [ 1] [a36n18:163107] *** Process received signal *** +[a36n18:163107] Signal: Segmentation fault (11) +[a36n18:163107] Signal code: (3) +[a36n18:163107] Failing at address: 0xb170000000000a0 +[a36n18:163107] [ 0] [0x1000000504d8] +[a36n18:163107] [ 1] [0x3ffffac27650] +[a36n18:163107] [ 2] [a35n10:151893] *** Process received signal *** +[a35n10:151893] Signal: Segmentation fault (11) +[a35n10:151893] Signal code: (3) +[a35n10:151893] Failing at address: 0x14170000000000a0 +[a35n10:151893] [ 0] [0x1000000504d8] +[a35n10:151893] [ 1] [0x3fffc54ce6d0] +[a35n10:151893] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n10:151893] [ 3] [a26n12:52453] *** Process received signal *** +[a26n12:52453] Signal: Segmentation fault (11) +[a26n12:52453] Signal code: (3) +[a26n12:52453] Failing at address: 0xb170000000000a0 +[a26n12:52453] [ 0] [0x1000000504d8] +[a26n12:52453] [ 1] [0x3fffcc3de270] +[a26n12:52453] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n12:52453] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n10:122210] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n10:122210] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n10:122210] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n10:122210] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n10:122210] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n10:122210] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n10:122210] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n04:175999] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n04:175999] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n15:148580] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n15:148580] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n12:148380] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n12:148380] *** End of error message *** +[b12n04:138158] *** Process received signal *** +[b12n04:138158] Signal: Segmentation fault (11) +[b12n04:138158] Signal code: (3) +[b12n04:138158] Failing at address: 0xb170000000000a0 +[b12n04:138158] [ 0] [0x1000000504d8] +[b12n04:138158] [ 1] [b08n13:158863] *** Process received signal *** +[b08n13:158863] Signal: Segmentation fault (11) +[b08n13:158863] Signal code: (3) +[b08n13:158863] Failing at address: 0x5170000000000a0 +[b08n13:158863] [ 0] [0x1000000504d8] +[b08n13:158863] [ 1] [0x3fffdd1b6480] +[b08n13:158863] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n13:158863] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n13:158863] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n13:158863] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n13:158863] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n13:158863] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n13:158863] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n13:158863] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n13:158863] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n13:158863] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n13:158863] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n13:158863] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n13:158863] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n13:158863] *** End of error message *** +[a31n12:156918] *** Process received signal *** +[a31n12:156918] Signal: Segmentation fault (11) +[a31n12:156918] Signal code: (3) +[a31n12:156918] Failing at address: 0x14170000000000a0 +[a31n12:156918] [ 0] [0x1000000504d8] +[a31n12:156918] [ 1] [0x3fffe3494a80] +[a31n12:156918] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n12:156918] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n12:156918] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n12:156918] [ 5] [b11n18:163572] *** Process received signal *** +[b11n18:163572] Signal: Segmentation fault (11) +[b11n18:163572] Signal code: (3) +[b11n18:163572] Failing at address: 0x14170000000000a0 +[b11n18:163572] [ 0] [0x1000000504d8] +[b11n18:163572] [ 1] [0x3fffc138f650] +[b11n18:163572] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n18:163572] [ 3] [b21n02:120190] *** Process received signal *** +[b21n02:120190] Signal: Segmentation fault (11) +[b21n02:120190] Signal code: (3) +[b21n02:120190] Failing at address: 0xb170000000000a0 +[b21n02:120190] [ 0] [0x1000000504d8] +[b21n02:120190] [ 1] [0x3fffc16b2440] +[b21n02:120190] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n02:120190] [ 3] [b06n02:163322] *** Process received signal *** +[b06n02:163322] Signal: Segmentation fault (11) +[b06n02:163322] Signal code: (3) +[b06n02:163322] Failing at address: 0x14170000000000a0 +[b06n02:163322] [ 0] [0x1000000504d8] +[b06n02:163322] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n15:177822] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n15:177822] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n15:177822] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n15:177822] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n15:177822] *** End of error message *** +[b10n08:129889] *** Process received signal *** +[b10n08:129889] Signal: Segmentation fault (11) +[b10n08:129889] Signal code: (3) +[b10n08:129889] Failing at address: 0x14170000000000a0 +[b10n08:129889] [ 0] [0x1000000504d8] +[b10n08:129889] [ 1] [0x3fffdeae1980] +[b10n08:129889] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n08:129889] [ 3] [a30n09:152062] *** Process received signal *** +[a30n09:152062] Signal: Segmentation fault (11) +[a30n09:152062] Signal code: (3) +[a30n09:152062] Failing at address: 0x14170000000000a0 +[a30n09:152062] [ 0] [0x1000000504d8] +[a30n09:152062] [ 1] [0x3ffff4922e00] +[a30n09:152062] [ 2] [b06n14:173896] *** Process received signal *** +[b06n14:173896] Signal: Segmentation fault (11) +[b06n14:173896] Signal code: (3) +[b06n14:173896] Failing at address: 0x14170000000000a0 +[b06n14:173896] [ 0] [0x1000000504d8] +[b06n14:173896] [ 1] [b06n01:165166] Signal: Segmentation fault (11) +[b06n01:165166] Signal code: (3) +[b06n01:165166] Failing at address: 0x5170000000000a0 +[b06n01:165166] [ 0] [0x1000000504d8] +[b06n01:165166] [ 1] [0x3fffe51bf950] +[b06n01:165166] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n09:122068] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n09:122068] [ 5] [b07n04:164046] *** Process received signal *** +[b07n04:164046] Signal: Segmentation fault (11) +[b07n04:164046] Signal code: (3) +[b07n04:164046] Failing at address: 0xb170000000000a0 +[b07n04:164046] [ 0] [0x1000000504d8] +[b07n04:164046] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n11:176299] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n11:176299] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n11:176299] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n11:176299] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n11:176299] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n03:168210] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n03:168210] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n03:168210] [ 6] [b09n06:167090] *** Process received signal *** +[b09n06:167090] Signal: Segmentation fault (11) +[b09n06:167090] Signal code: (3) +[b09n06:167090] Failing at address: 0x14170000000000a0 +[b09n06:167090] [ 0] [0x1000000504d8] +[b09n06:167090] [ 1] [a30n06:155087] *** Process received signal *** +[a30n06:155087] Signal: Segmentation fault (11) +[a30n06:155087] Signal code: (3) +[a30n06:155087] Failing at address: 0x14170000000000a0 +[a30n06:155087] [ 0] [0x1000000504d8] +[a30n06:155087] [ 1] [0x3fffd8a4fde0] +[a30n06:155087] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n06:155087] [ 3] [ 1] [0x3fffd4e68390] +[a30n08:163101] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n08:163101] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n08:163101] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n08:163101] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n08:163101] [ 6] [a34n11:147712] *** Process received signal *** +[a34n11:147712] Signal: Segmentation fault (11) +[a34n11:147712] Signal code: (3) +[a34n11:147712] Failing at address: 0x14170000000000a0 +[a34n11:147712] [ 0] [0x1000000504d8] +[a34n11:147712] [ 1] [0x3fffebf50970] +[a34n11:147712] [ 2] [b01n16:166690] *** Process received signal *** +[b01n16:166690] Signal: Segmentation fault (11) +[b01n16:166690] Signal code: (3) +[b01n16:166690] Failing at address: 0xb170000000000a0 +[b01n16:166690] [ 0] [0x1000000504d8] +[b01n16:166690] [ 1] [0x3fffc813f380] +[b01n16:166690] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n16:166690] [ 3] [b14n07:127354] *** Process received signal *** +[b14n07:127354] Signal: Segmentation fault (11) +[b14n07:127354] Signal code: (3) +[b14n07:127354] Failing at address: 0x14170000000000a0 +[b14n07:127354] [ 0] [0x1000000504d8] +[b14n07:127354] [ 1] [0x3fffc1bb6620] +[b14n07:127354] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n07:127354] [ 3] [a31n08:156372] *** Process received signal *** +[a31n08:156372] Signal: Segmentation fault (11) +[a31n08:156372] Signal code: (3) +[a31n08:156372] Failing at address: 0x14170000000000a0 +[a31n08:156372] [ 0] [0x1000000504d8] +[a31n08:156372] [ 1] [0x3fffc2ef3030] +[a31n08:156372] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n08:156372] [ 3] [b01n15:03734] *** Process received signal *** +[b01n15:03734] Signal: Segmentation fault (11) +[b01n15:03734] Signal code: (3) +[b01n15:03734] Failing at address: 0x14170000000000a0 +[b01n15:03734] [ 0] [0x1000000504d8] +[b01n15:03734] [ 1] [0x3ffff79edae0] +[b01n15:03734] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n15:03734] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n13:163196] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n13:163196] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n13:165260] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n13:165260] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n13:165260] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n13:165260] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n13:165260] [11] [a27n06:60152] *** Process received signal *** +[a27n06:60152] Signal: Segmentation fault (11) +[a27n06:60152] Signal code: (3) +[a27n06:60152] Failing at address: 0x14170000000000a0 +[a27n06:60152] [ 0] [0x1000000504d8] +[a27n06:60152] [ 1] [0x3ffffbbb94b0] +[a27n06:60152] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n06:60152] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n06:60152] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n06:60152] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n06:60152] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n06:60152] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n06:60152] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n06:60152] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n06:60152] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n06:60152] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n06:60152] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n06:60152] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n06:60152] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n06:60152] *** End of error message *** +[a29n09:79007] *** Process received signal *** +[a29n09:79007] Signal: Segmentation fault (11) +[a29n09:79007] Signal code: (3) +[a29n09:79007] Failing at address: 0xb170000000000a0 +[a29n09:79007] [ 0] [0x1000000504d8] +[a29n09:79007] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n02:155384] *** End of error message *** +[a34n05:148377] *** Process received signal *** +[a34n05:148377] Signal: Segmentation fault (11) +[a34n05:148377] Signal code: (3) +[a34n05:148377] Failing at address: 0x14170000000000a0 +[a34n05:148377] [ 0] [0x1000000504d8] +[a34n05:148377] [ 1] [0x1000000504d8] +[b17n07:125472] [ 1] [0x3fffc8a6c6e0] +[b17n07:125472] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n07:125472] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n07:125472] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n07:125472] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n07:125472] [ 6] [a34n08:163161] *** Process received signal *** +[a34n08:163161] Signal: Segmentation fault (11) +[a34n08:163161] Signal code: (3) +[a34n08:163161] Failing at address: 0xb170000000000a0 +[a34n08:163161] [ 0] [0x1000000504d8] +[a34n08:163161] [ 1] [0x3fffceb98730] +[a34n08:163161] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n08:163161] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n08:163161] [ 4] [b22n11:120242] [ 0] [0x1000000504d8] +[b22n11:120242] [ 1] [0x3fffc6d18e30] +[b22n11:120242] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n11:120242] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n11:120242] [ 4] [a35n18:145742] *** Process received signal *** +[a35n18:145742] Signal: Segmentation fault (11) +[a35n18:145742] Signal code: (3) +[a35n18:145742] Failing at address: 0xb170000000000a0 +[a35n18:145742] [ 0] [0x1000000504d8] +[a35n18:145742] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n11:175598] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n11:175598] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n11:175598] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n11:175598] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n11:175598] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n11:175598] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n11:175598] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n11:175598] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n11:175598] [13] [b10n04:169413] *** Process received signal *** +[b10n04:169413] Signal: Segmentation fault (11) +[b10n04:169413] Signal code: (3) +[b10n04:169413] Failing at address: 0xb170000000000a0 +[b10n04:169413] [ 0] [0x1000000504d8] +[b10n04:169413] [ 1] [0x3fffe7b1f5c0] +[b10n04:169413] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n04:169413] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n04:169413] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n04:169413] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n04:169413] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n04:169413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n04:169413] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n04:169413] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n04:169413] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n04:169413] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n04:169413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n04:169413] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n04:169413] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n04:169413] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n12:177118] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n12:177118] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n12:177118] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n12:177118] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n12:177118] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n12:177118] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n12:177118] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n12:177118] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n15:123040] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n15:123040] [ 5] [0x1000000504d8] +[b23n12:115637] [ 1] [0x3ffffc2a2030] +[b23n12:115637] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n12:115637] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n12:115637] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n12:115637] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n15:166706] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n15:166706] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n15:166706] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n15:166706] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n15:166706] [ 9] [a02n06:38118] *** Process received signal *** +[a02n06:38118] Signal: Segmentation fault (11) +[a02n06:38118] Signal code: (3) +[a02n06:38118] Failing at address: 0x14170000000000a0 +[a02n06:38118] [ 0] [0x1000000504d8] +[a02n06:38118] [ 1] [0x3fffd6ad28d0] +[a02n06:38118] [ 2] [0x3ffff19a24d0] +[b11n12:154341] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n12:154341] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n12:154341] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n12:154341] [ 5] [a30n15:154386] *** Process received signal *** +[a30n15:154386] Signal: Segmentation fault (11) +[a30n15:154386] Signal code: (3) +[a30n15:154386] Failing at address: 0x14170000000000a0 +[a30n15:154386] [ 0] [0x1000000504d8] +[a30n15:154386] [ 1] [0x3fffcce26e50] +[a30n15:154386] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n06:177848] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n06:177848] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n06:177848] [11] [b19n08:121687] *** Process received signal *** +[b19n08:121687] Signal: Segmentation fault (11) +[b19n08:121687] Signal code: (3) +[b19n08:121687] Failing at address: 0x14170000000000a0 +[b19n08:121687] [ 0] [0x1000000504d8] +[b19n08:121687] [ 1] [0x3fffc73c52d0] +[b19n08:121687] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n08:121687] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n06:10224] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n06:10224] [ 6] [b02n03:39686] *** Process received signal *** +[b02n03:39686] Signal: Segmentation fault (11) +[b02n03:39686] Signal code: (3) +[b02n03:39686] Failing at address: 0x14170000000000a0 +[b02n03:39686] [ 0] [0x1000000504d8] +[b02n03:39686] [ 1] [0x3fffdac41060] +[b02n03:39686] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n03:39686] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n03:39686] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n03:39686] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n03:39686] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n03:39686] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n03:39686] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n03:39686] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n03:39686] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n03:39686] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n03:39686] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n03:39686] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n03:39686] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n03:39686] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n03:135166] *** End of error message *** +[0x1000000504d8] +[b17n15:137007] [ 1] [0x3fffdc0b4ff0] +[b17n15:137007] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n15:137007] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n15:137007] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n15:137007] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n05:169787] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n05:169787] [10] [a35n01:138003] *** Process received signal *** +[a35n01:138003] Signal: Segmentation fault (11) +[a35n01:138003] Signal code: (3) +[a35n01:138003] Failing at address: 0xb170000000000a0 +[a35n01:138003] [ 0] [0x1000000504d8] +[a35n01:138003] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n14:163551] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n14:163551] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n14:163551] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n14:163551] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n14:163551] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n14:163551] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n12:140120] [ 5] [a32n11:152813] *** Process received signal *** +[a32n11:152813] Signal: Segmentation fault (11) +[a32n11:152813] Signal code: (3) +[a32n11:152813] Failing at address: 0x14170000000000a0 +[a32n11:152813] [ 0] [0x1000000504d8] +[a32n11:152813] [ 1] [0x3fffd16349f0] +[a32n11:152813] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n11:152813] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n11:152813] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n11:152813] [ 5] [a31n09:164363] *** Process received signal *** +[a31n09:164363] Signal: Segmentation fault (11) +[a31n09:164363] Signal code: (3) +[a31n09:164363] Failing at address: 0xb170000000000a0 +[a31n09:164363] [ 0] [0x1000000504d8] +[a31n09:164363] [ 1] [0x3fffdfac1fc0] +[a31n09:164363] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n09:164363] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n03:168930] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n03:168930] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n03:168930] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n03:168930] [ 8] [b02n11:02684] *** Process received signal *** +[b02n11:02684] Signal: Segmentation fault (11) +[b02n11:02684] Signal code: (3) +[b02n11:02684] Failing at address: 0x14170000000000a0 +[b02n11:02684] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n02:49107] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n02:49107] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n02:49107] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n02:49107] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n02:49107] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n02:49107] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n02:49107] *** End of error message *** +[a30n03:167779] *** Process received signal *** +[a30n03:167779] Signal: Segmentation fault (11) +[a30n03:167779] Signal code: (3) +[a30n03:167779] Failing at address: 0x14170000000000a0 +[a30n03:167779] [ 0] [0x1000000504d8] +[a30n03:167779] [ 1] [0x3ffff0b35830] +[a30n03:167779] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n03:167779] [ 3] [0x1000000504d8] +[b14n17:119468] [ 1] [0x3ffff4b1abf0] +[b14n17:119468] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n17:119468] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n17:119468] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n12:11420] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n12:11420] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n15:161755] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n15:161755] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n13:06928] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n17:146798] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n17:146798] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n17:146798] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n17:146798] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n17:146798] [ 9] [a29n01:153682] *** Process received signal *** +[a29n01:153682] Signal: Segmentation fault (11) +[a29n01:153682] Signal code: (3) +[a29n01:153682] Failing at address: 0x14170000000000a0 +[a29n01:153682] [ 0] [0x1000000504d8] +[a29n01:153682] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n17:126527] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n17:126527] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n16:124798] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n16:124798] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n16:124798] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n16:124798] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n16:124798] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n16:124798] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n16:124798] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n16:124798] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n16:124798] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n09:147795] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n09:147795] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n09:147795] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n09:147795] *** End of error message *** +[b07n06:177003] *** Process received signal *** +[b07n06:177003] Signal: Segmentation fault (11) +[b07n06:177003] Signal code: (3) +[b07n06:177003] Failing at address: 0x5170000000000a0 +[b07n06:177003] [ 0] [a27n14:21316] *** Process received signal *** +[a27n14:21316] Signal: Segmentation fault (11) +[a27n14:21316] Signal code: (3) +[a27n14:21316] Failing at address: 0xb170000000000a0 +[a27n14:21316] [ 0] [0x1000000504d8] +[a27n14:21316] [ 1] [a35n02:149226] *** Process received signal *** +[a35n02:149226] Signal: Segmentation fault (11) +[a35n02:149226] Signal code: (3) +[a35n02:149226] Failing at address: 0xb170000000000a0 +[a35n02:149226] [ 0] [0x1000000504d8] +[a35n02:149226] [ 1] [a26n01:65729] *** Process received signal *** +[a26n01:65729] Signal: Segmentation fault (11) +[a26n01:65729] Signal code: (3) +[a26n01:65729] Failing at address: 0x14170000000000a0 +[a26n01:65729] [ 0] [0x1000000504d8] +[a26n01:65729] [ 1] [0x3fffd1f466e0] +[a26n01:65729] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n01:65729] [ 3] [b18n03:136667] *** Process received signal *** +[b18n03:136667] Signal: Segmentation fault (11) +[b18n03:136667] Signal code: (3) +[b18n03:136667] Failing at address: 0xb170000000000a0 +[b18n03:136667] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n03:125240] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n06:154622] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n06:154622] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n06:154622] [ 9] [b06n05:163960] *** Process received signal *** +[b06n05:163960] Signal: Segmentation fault (11) +[b06n05:163960] Signal code: (3) +[b06n05:163960] Failing at address: 0x14170000000000a0 +[b06n05:163960] [ 0] [0x1000000504d8] +[b06n05:163960] [ 1] [0x3fffeccb5d60] +[b06n05:163960] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n05:163960] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n05:163960] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n05:163960] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n05:163960] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n05:163960] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n05:163960] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n05:163960] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n05:163960] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n05:163960] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n05:163960] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n05:163960] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n05:163960] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n05:163960] *** End of error message *** +[b01n04:169707] *** Process received signal *** +[b01n04:169707] Signal: Segmentation fault (11) +[b01n04:169707] Signal code: (3) +[b01n04:169707] Failing at address: 0x5170000000000a0 +[b01n04:169707] [ 0] [0x1000000504d8] +[b01n04:169707] [ 1] [0x3fffca6f5150] +[b01n04:169707] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n04:169707] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n04:169707] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n04:169707] [ 5] [b01n03:01352] *** Process received signal *** +[b01n03:01352] Signal: Segmentation fault (11) +[b01n03:01352] Signal code: (3) +[b01n03:01352] Failing at address: 0xb170000000000a0 +[b01n03:01352] [ 0] [0x1000000504d8] +[b01n03:01352] [ 1] [0x3fffc46fa420] +[b01n03:01352] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n03:01352] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n02:127220] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n02:127220] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n02:127220] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n09:121728] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n09:121728] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n09:121728] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n09:121728] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n09:121728] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n09:121728] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n09:121728] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n09:121728] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n09:121728] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n09:121728] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n09:121728] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n09:121728] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n11:148673] *** End of error message *** +[b19n05:131261] *** Process received signal *** +[b19n05:131261] Signal: Segmentation fault (11) +[b19n05:131261] Signal code: (3) +[b19n05:131261] Failing at address: 0xb170000000000a0 +[b19n05:131261] [ 0] [b19n02:123707] *** Process received signal *** +[b19n02:123707] Signal: Segmentation fault (11) +[b19n02:123707] Signal code: (3) +[b19n02:123707] Failing at address: 0x14170000000000a0 +[b19n02:123707] [ 0] [0x1000000504d8] +[b19n02:123707] [ 1] [0x3ffff6821090] +[b19n02:123707] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n02:123707] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n05:162565] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n05:162565] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n05:162565] [ 9] [b06n17:163816] Signal: Segmentation fault (11) +[b06n17:163816] Signal code: (3) +[b06n17:163816] Failing at address: 0xb170000000000a0 +[b06n17:163816] [b21n04:118344] *** Process received signal *** +[b21n04:118344] Signal: Segmentation fault (11) +[b21n04:118344] Signal code: (3) +[b21n04:118344] Failing at address: 0x14170000000000a0 +[b21n04:118344] [ 0] [0x1000000504d8] +[b21n04:118344] [ 1] [0x3fffdda018a0] +[b21n04:118344] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n04:118344] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n04:118344] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n04:118344] [ 5] [a26n04:54379] *** Process received signal *** +[a26n04:54379] Signal: Segmentation fault (11) +[a26n04:54379] Signal code: (3) +[a26n04:54379] Failing at address: 0x14170000000000a0 +[a26n04:54379] [ 0] [b18n05:135122] *** Process received signal *** +[b18n05:135122] Signal: Segmentation fault (11) +[b18n05:135122] Signal code: (3) +[b18n05:135122] Failing at address: 0xb170000000000a0 +[b18n05:135122] [ 0] [0x1000000504d8] +[b18n05:135122] [ 1] [0x3ffff084cd80] +[b18n05:135122] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n05:135122] [ 3] [b01n07:03459] *** Process received signal *** +[b01n07:03459] Signal: Segmentation fault (11) +[b01n07:03459] Signal code: (3) +[b01n07:03459] Failing at address: 0x14170000000000a0 +[b01n07:03459] [ 0] [0x1000000504d8] +[b01n07:03459] [ 1] [0x3fffeec409b0] +[b01n07:03459] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n07:03459] [ 3] [0x1000000504d8] +[a34n01:164963] [ 1] [0x3fffc75fe840] +[a34n01:164963] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n01:164963] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n01:164963] [ 4] [b01n08:172426] *** Process received signal *** +[b01n08:172426] Signal: Segmentation fault (11) +[b01n08:172426] Signal code: (3) +[b01n08:172426] Failing at address: 0x14170000000000a0 +[b01n08:172426] [ 0] [0x1000000504d8] +[b01n08:172426] [ 1] [0x3fffde6e9bf0] +[b01n08:172426] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n06:130543] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n06:130543] [ 5] [b07n09:163515] *** Process received signal *** +[b07n09:163515] Signal: Segmentation fault (11) +[b07n09:163515] Signal code: (3) +[b07n09:163515] Failing at address: 0xb170000000000a0 +[b07n09:163515] [ 0] [0x1000000504d8] +[b07n09:163515] [ 1] [0x3fffec396d40] +[b07n09:163515] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n09:163515] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n09:163515] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n09:163515] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n09:163515] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n09:163515] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n09:163515] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n09:163515] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n09:163515] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n09:163515] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n09:163515] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n09:163515] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n09:163515] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n09:163515] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n02:122560] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n02:122560] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n02:122560] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n02:122560] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n02:122560] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n02:122560] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n02:122560] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n02:122560] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n02:122560] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n10:161719] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n10:161719] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n10:161719] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n10:161719] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n17:162361] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n17:162361] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n17:162361] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n17:162361] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n17:162361] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n17:162361] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n17:162361] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n17:162361] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n17:162361] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n17:162361] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n17:162361] [13] [b18n06:133952] *** Process received signal *** +[b18n06:133952] Signal: Segmentation fault (11) +[b18n06:133952] Signal code: (3) +[b18n06:133952] Failing at address: 0x14170000000000a0 +[b18n06:133952] [ 0] [b08n09:174688] *** Process received signal *** +[b08n09:174688] Signal: Segmentation fault (11) +[b08n09:174688] Signal code: (3) +[b08n09:174688] Failing at address: 0xb170000000000a0 +[b08n09:174688] [ 0] [0x1000000504d8] +[b08n09:174688] [ 1] [0x3fffd6772b50] +[b08n09:174688] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n09:174688] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n09:174688] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n09:174688] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n09:174688] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n09:174688] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n09:174688] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n09:174688] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n09:174688] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n09:174688] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n09:174688] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n09:174688] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n09:174688] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n09:174688] *** End of error message *** +[a28n04:13081] *** Process received signal *** +[a28n04:13081] Signal: Segmentation fault (11) +[a28n04:13081] Signal code: (3) +[a28n04:13081] Failing at address: 0xb170000000000a0 +[a28n04:13081] [a28n02:178474] *** Process received signal *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n03:151884] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n03:151884] *** End of error message *** +[a28n01:179262] *** Process received signal *** +[a28n01:179262] Signal: Segmentation fault (11) +[a28n01:179262] Signal code: (3) +[a28n01:179262] Failing at address: 0x14170000000000a0 +[a28n01:179262] [ 0] [0x1000000504d8] +[a28n01:179262] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n08:175808] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n08:175808] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n08:175808] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n08:175808] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n08:175808] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n08:175808] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n08:175808] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n08:175808] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n08:175808] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n08:175808] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n08:175808] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n08:175808] *** End of error message *** +[b08n11:164995] *** Process received signal *** +[b08n11:164995] Signal: Segmentation fault (11) +[b08n11:164995] Signal code: (3) +[b08n11:164995] Failing at address: 0x5170000000000a0 +[b08n11:164995] [ 0] [0x1000000504d8] +[b08n11:164995] [ 1] [0x3fffd0acf7e0] +[b08n11:164995] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n11:164995] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n11:164995] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n11:164995] [ 5] [a36n01:141340] *** Process received signal *** +[a36n01:141340] Signal: Segmentation fault (11) +[a36n01:141340] Signal code: (3) +[a36n01:141340] Failing at address: 0xb170000000000a0 +[a36n01:141340] [ 0] [a36n04:142256] *** Process received signal *** +[a36n04:142256] Signal: Segmentation fault (11) +[a36n04:142256] Signal code: (3) +[a36n04:142256] Failing at address: 0xb170000000000a0 +[a36n04:142256] [ 0] [0x1000000504d8] +[a36n04:142256] [ 1] [0x3ffff6cb2e50] +[a36n04:142256] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n04:142256] [ 3] [a36n02:147884] *** Process received signal *** +[a36n02:147884] Signal: Segmentation fault (11) +[a36n02:147884] Signal code: (3) +[a36n02:147884] Failing at address: 0x5170000000000a0 +[a36n02:147884] [ 0] [a36n03:153854] *** Process received signal *** +[a36n03:153854] Signal: Segmentation fault (11) +[a36n03:153854] Signal code: (3) +[a36n03:153854] Failing at address: 0x5170000000000a0 +[a36n03:153854] [ 0] [0x1000000504d8] +[a36n03:153854] [ 1] [0x3ffff8985790] +[a36n03:153854] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n03:153854] [ 3] [b21n05:130805] *** Process received signal *** +[b21n05:130805] Signal: Segmentation fault (11) +[b21n05:130805] Signal code: (3) +[b21n05:130805] Failing at address: 0xb170000000000a0 +[b21n05:130805] [ 0] [0x1000000504d8] +[b21n05:130805] [ 1] [0x3fffd76e89b0] +[b21n05:130805] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n05:130805] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n05:130805] [ 4] [b01n01:146655] *** Process received signal *** +[b01n01:146655] Signal: Segmentation fault (11) +[b01n01:146655] Signal code: (3) +[b01n01:146655] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n06:162918] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n06:162918] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n06:162918] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n06:162918] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n06:162918] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n17:118573] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n17:118573] *** End of error message *** +[a36n14:152039] *** Process received signal *** +[a36n14:152039] Signal: Segmentation fault (11) +[a36n14:152039] Signal code: (3) +[a36n14:152039] Failing at address: 0x14170000000000a0 +[a36n14:152039] [ 0] [0x1000000504d8] +[a36n14:152039] [ 1] [0x3ffff6c106f0] +[a36n14:152039] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n14:152039] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n14:152039] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n14:152039] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n14:152039] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n14:152039] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n14:152039] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n14:152039] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n14:152039] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n14:152039] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n14:152039] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n14:152039] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n14:152039] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n14:152039] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n09:140256] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n09:140256] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n09:140256] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n09:140256] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n09:140256] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n09:140256] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n09:140256] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n09:140256] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n09:140256] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n09:140256] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n09:140256] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n10:164939] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n10:164939] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n01:171784] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n01:171784] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n01:171784] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n01:171784] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n01:171784] [14] [a26n05:54535] *** Process received signal *** +[a26n05:54535] Signal: Segmentation fault (11) +[a26n05:54535] Signal code: (3) +[a26n05:54535] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n16:121928] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n16:121928] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n16:121928] [ 6] [a32n05:154185] *** Process received signal *** +[a32n05:154185] Signal: Segmentation fault (11) +[a32n05:154185] Signal code: (3) +[a32n05:154185] Failing at address: 0xb170000000000a0 +[a32n05:154185] [ 0] [0x1000000504d8] +[a32n05:154185] [ 1] [0x3fffc83e6ef0] +[a32n05:154185] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n05:154185] [ 3] [b16n02:121683] *** Process received signal *** +[b16n02:121683] Signal: Segmentation fault (11) +[b16n02:121683] Signal code: (3) +[b16n02:121683] Failing at address: 0x14170000000000a0 +[b16n02:121683] [ 0] [0x1000000504d8] +[b16n02:121683] [ 1] [0x3fffeb98f430] +[b16n02:121683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n02:121683] [ 3] [a32n01:158306] *** Process received signal *** +[a32n01:158306] Signal: Segmentation fault (11) +[a32n01:158306] Signal code: (3) +[a32n01:158306] Failing at address: 0x14170000000000a0 +[a32n01:158306] [ 0] [0x1000000504d8] +[a32n01:158306] [ 1] [0x3fffc5f8d8f0] +[a32n01:158306] [ 2] [a28n16:171167] *** Process received signal *** +[a28n16:171167] Signal: Segmentation fault (11) +[a28n16:171167] Signal code: (3) +[a28n16:171167] Failing at address: 0x14170000000000a0 +[a28n16:171167] [ 0] [0x1000000504d8] +[a28n16:171167] [ 1] [b21n18:118680] *** Process received signal *** +[b21n18:118680] Signal: Segmentation fault (11) +[b21n18:118680] Signal code: (3) +[b21n18:118680] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n08:164751] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n08:164751] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n08:164751] [ 9] [b16n04:123054] *** Process received signal *** +[b16n04:123054] Signal: Segmentation fault (11) +[b16n04:123054] Signal code: (3) +[b16n04:123054] Failing at address: 0x14170000000000a0 +[b16n04:123054] [ 0] [0x1000000504d8] +[b16n04:123054] [ 1] [0x3ffff50c9e20] +[b16n04:123054] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n04:123054] [ 3] [b16n01:132063] *** Process received signal *** +[b16n01:132063] Signal: Segmentation fault (11) +[b16n01:132063] Signal code: (3) +[b16n01:132063] Failing at address: 0x14170000000000a0 +[b16n01:132063] [ 0] [0x1000000504d8] +[b16n01:132063] [ 1] [0x3fffc17b6cc0] +[b16n01:132063] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n01:132063] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n06:133057] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n06:133057] *** End of error message *** +[b03n04:04850] *** Process received signal *** +[b03n04:04850] Signal: Segmentation fault (11) +[b03n04:04850] Signal code: (3) +[b03n04:04850] Failing at address: 0xb170000000000a0 +[b03n04:04850] [ 0] [0x1000000504d8] +[b03n04:04850] [ 1] [0x3ffff391a040] +[b03n04:04850] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n04:04850] [ 3] [b16n03:121809] *** Process received signal *** +[b16n03:121809] Signal: Segmentation fault (11) +[b16n03:121809] Signal code: (3) +[b16n03:121809] Failing at address: 0xb170000000000a0 +[b16n03:121809] [ 0] [0x1000000504d8] +[b16n03:121809] [ 1] [0x3fffdae53820] +[b16n03:121809] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n03:121809] [ 3] [b16n05:122271] *** Process received signal *** +[b16n05:122271] Signal: Segmentation fault (11) +[b16n05:122271] Signal code: (3) +[b16n05:122271] Failing at address: 0x14170000000000a0 +[b16n05:122271] [ 0] [0x1000000504d8] +[b16n05:122271] [ 1] [0x3fffc5591e80] +[b16n05:122271] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n05:122271] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n04:133902] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n04:133902] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n04:133902] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n04:133902] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n04:133902] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n04:133902] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n04:133902] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n04:133902] *** End of error message *** +[b16n06:132106] *** Process received signal *** +[b16n06:132106] Signal: Segmentation fault (11) +[b16n06:132106] Signal code: (3) +[b16n06:132106] Failing at address: 0xb170000000000a0 +[b16n06:132106] [ 0] [0x1000000504d8] +[b16n06:132106] [ 1] [0x3fffcb87b7a0] +[b16n06:132106] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n06:132106] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n05:173081] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n05:173081] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n02:116889] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n02:116889] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n02:116889] *** End of error message *** +[b16n07:119635] *** Process received signal *** +[b16n07:119635] Signal: Segmentation fault (11) +[b16n07:119635] Signal code: (3) +[b16n07:119635] Failing at address: 0xb170000000000a0 +[b16n07:119635] [ 0] [0x1000000504d8] +[b16n07:119635] [ 1] [0x3fffe88799b0] +[b16n07:119635] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n07:119635] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n18:162421] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n18:162421] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n18:162421] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n18:162421] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n18:162421] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n18:162421] [10] [b03n10:02711] *** Process received signal *** +[b01n06:171503] *** Process received signal *** +[b01n06:171503] Signal: Segmentation fault (11) +[b01n06:171503] Signal code: (3) +[b01n06:171503] Failing at address: 0x5170000000000a0 +[b01n06:171503] [ 0] [0x1000000504d8] +[b01n06:171503] [ 1] [0x3fffd41c53e0] +[b01n06:171503] [ 2] [b15n06:120735] *** Process received signal *** +[b15n06:120735] Signal: Segmentation fault (11) +[b15n06:120735] Signal code: (3) +[b15n06:120735] Failing at address: 0xb170000000000a0 +[b15n06:120735] [ 0] [0x1000000504d8] +[b15n06:120735] [ 1] [0x3ffff4bea2a0] +[b15n06:120735] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n06:120735] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n06:120735] [ 4] [b15n05:120332] *** Process received signal *** +[b15n05:120332] Signal: Segmentation fault (11) +[b15n05:120332] Signal code: (3) +[b15n05:120332] Failing at address: 0x14170000000000a0 +[b15n05:120332] [ 0] [0x1000000504d8] +[b15n05:120332] [ 1] [0x3fffeb83a050] +[b15n05:120332] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n05:120332] [ 3] [b03n07:03372] *** Process received signal *** +[b03n07:03372] Signal: Segmentation fault (11) +[b03n07:03372] Signal code: (3) +[b03n07:03372] Failing at address: 0xb170000000000a0 +[b02n15:169494] *** Process received signal *** +[b02n15:169494] Signal: Segmentation fault (11) +[b02n15:169494] Signal code: (3) +[b02n15:169494] Failing at address: 0x14170000000000a0 +[b02n15:169494] [ 0] [0x1000000504d8] +[b02n15:169494] [ 1] [0x3ffffa1c0640] +[b02n15:169494] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n11:160797] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n06:163677] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n06:163677] [ 6] [b01n05:02909] *** Process received signal *** +[b01n05:02909] Signal: Segmentation fault (11) +[b01n05:02909] Signal code: (3) +[b01n05:02909] Failing at address: 0x14170000000000a0 +[b01n05:02909] [ 0] [0x1000000504d8] +[b01n05:02909] [ 1] [0x3fffc70ead50] +[b01n05:02909] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n05:02909] [ 3] [b14n03:130346] *** Process received signal *** +[b14n03:130346] Signal: Segmentation fault (11) +[b14n03:130346] Signal code: (3) +[b14n03:130346] Failing at address: 0x14170000000000a0 +[b14n03:130346] [ 0] [0x1000000504d8] +[b14n03:130346] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n07:60172] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n07:60172] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n07:60172] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n07:60172] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n07:60172] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n07:60172] *** End of error message *** +[b14n02:139023] *** Process received signal *** +[b14n02:139023] Signal: Segmentation fault (11) +[b14n02:139023] Signal code: (3) +[b14n02:139023] Failing at address: 0xb170000000000a0 +[b14n02:139023] [ 0] [0x1000000504d8] +[b14n02:139023] [ 1] [0x3ffff25f2410] +[b14n02:139023] [ 2] [b14n01:128004] *** Process received signal *** +[b14n01:128004] Signal: Segmentation fault (11) +[b14n01:128004] Signal code: (3) +[b14n01:128004] Failing at address: 0xb170000000000a0 +[b14n01:128004] [ 0] [0x1000000504d8] +[b14n01:128004] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n12:138737] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n12:138737] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n12:138737] [ 9] [b16n08:121434] *** Process received signal *** +[b16n08:121434] Signal: Segmentation fault (11) +[b16n08:121434] Signal code: (3) +[b16n08:121434] Failing at address: 0x14170000000000a0 +[b16n08:121434] [ 0] [0x1000000504d8] +[b16n08:121434] [ 1] [0x3ffff4373ce0] +[b16n08:121434] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n08:121434] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n15:139059] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n15:139059] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n15:139059] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n15:139059] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n15:139059] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n15:139059] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n15:139059] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n15:139059] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n16:164551] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n16:164551] *** End of error message *** +[b14n04:126860] *** Process received signal *** +[b14n04:126860] Signal: Segmentation fault (11) +[b14n04:126860] Signal code: (3) +[b14n04:126860] Failing at address: 0x14170000000000a0 +[b15n07:124352] *** Process received signal *** +[b15n07:124352] Signal: Segmentation fault (11) +[b15n07:124352] Signal code: (3) +[b15n07:124352] Failing at address: 0xb170000000000a0 +[b15n07:124352] [b14n05:126782] *** Process received signal *** +[b14n05:126782] Signal: Segmentation fault (11) +[b14n05:126782] Signal code: (3) +[b14n05:126782] Failing at address: 0xb170000000000a0 +[b14n05:126782] [ 0] [0x1000000504d8] +[b14n05:126782] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n17:176414] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n17:176414] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n09:155757] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n09:155757] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n09:155757] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n09:155757] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n09:155757] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n17:123238] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n17:123238] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n17:123238] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n17:123238] *** End of error message *** +[b05n13:10199] *** Process received signal *** +[b05n13:10199] Signal: Segmentation fault (11) +[b05n13:10199] Signal code: (3) +[b05n13:10199] Failing at address: 0x14170000000000a0 +[b05n13:10199] [ 0] [0x1000000504d8] +[b05n13:10199] [ 1] [0x3fffe29ab5c0] +[b05n13:10199] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n13:10199] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n13:10199] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n13:10199] [ 5] [a33n06:165302] *** Process received signal *** +[a33n06:165302] Signal: Segmentation fault (11) +[a33n06:165302] Signal code: (3) +[a33n06:165302] Failing at address: 0x5170000000000a0 +[a33n06:165302] [ 0] [0x1000000504d8] +[a33n06:165302] [ 1] [0x3fffdbd6de90] +[a33n06:165302] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n06:165302] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n06:165302] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n06:165302] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n06:165302] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n06:165302] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n06:165302] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n06:165302] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n06:165302] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n06:165302] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n06:165302] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n06:165302] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n06:165302] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n06:165302] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n06:157790] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n06:157790] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n06:157790] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n06:157790] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n06:157790] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n06:157790] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n06:157790] *** End of error message *** +[b10n01:165127] *** Process received signal *** +[b10n01:165127] Signal: Segmentation fault (11) +[b10n01:165127] Signal code: (3) +[b10n01:165127] Failing at address: 0x14170000000000a0 +[b10n01:165127] [ 0] [0x1000000504d8] +[b10n01:165127] [ 1] [0x3fffe5251010] +[b10n01:165127] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n01:165127] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n06:138766] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n06:138766] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n06:138766] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n06:138766] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n06:138766] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n06:138766] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n11:117475] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n11:117475] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n11:117475] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n11:117475] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n06:161720] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n06:161720] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n06:161720] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n06:161720] [10] [a33n17:168151] *** Process received signal *** +[a33n17:168151] Signal: Segmentation fault (11) +[a33n17:168151] Signal code: (3) +[a33n17:168151] Failing at address: 0xb170000000000a0 +[a33n17:168151] [ 0] [b08n18:164007] *** Process received signal *** +[b08n18:164007] Signal: Segmentation fault (11) +[b08n18:164007] Signal code: (3) +[b08n18:164007] Failing at address: 0x14170000000000a0 +[b08n18:164007] [ 0] [0x1000000504d8] +[b08n18:164007] [ 1] [0x3fffe6517200] +[b08n18:164007] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n18:164007] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n18:164007] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n18:164007] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n18:164007] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n18:164007] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n18:164007] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n18:164007] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n18:164007] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n18:164007] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n18:164007] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n18:164007] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n18:164007] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n18:164007] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n06:117525] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n06:117525] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n06:117525] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n06:117525] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n07:152085] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n07:152085] *** End of error message *** +[b11n03:154294] [ 0] [0x1000000504d8] +[b11n03:154294] [ 1] [0x3fffea77dce0] +[b11n03:154294] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n03:154294] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n03:154294] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n13:171109] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n13:171109] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n13:171109] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n13:171109] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n12:178597] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n12:178597] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n12:178597] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n12:141259] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n12:141259] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n12:141259] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n12:141259] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n12:141259] [10] [a31n04:158636] *** Process received signal *** +[a31n04:158636] Signal: Segmentation fault (11) +[a31n04:158636] Signal code: (3) +[a31n04:158636] Failing at address: 0xb170000000000a0 +[a31n04:158636] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n14:178327] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n07:168964] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n07:168964] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n07:168964] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n07:168964] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n17:176827] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n17:176827] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n17:176827] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n17:176827] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n17:176827] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n17:176827] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n17:176827] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n17:176827] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n17:176827] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n17:176827] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n17:176827] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n10:16975] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n10:16975] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n09:119634] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n09:119634] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n11:120581] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n11:120581] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n11:120581] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n04:145750] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n04:145750] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n04:145750] [13] [a33n05:139434] *** Process received signal *** +[a33n05:139434] Signal: Segmentation fault (11) +[a33n05:139434] Signal code: (3) +[a33n05:139434] Failing at address: 0xb170000000000a0 +[a33n05:139434] [ 0] [0x1000000504d8] +[a33n05:139434] [ 1] [0x3fffdb55e760] +[a33n05:139434] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n05:139434] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n17:169737] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n17:169737] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n17:169737] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n17:169737] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n17:169737] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n17:169737] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n17:169737] [13] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n17:177404] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n17:177404] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n17:177404] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n17:177404] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n17:177404] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n17:177404] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n17:177404] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n17:177404] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n10:11489] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n10:11489] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n10:11489] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n10:11489] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n10:11489] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n14:08398] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n13:153678] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n13:153678] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n13:153678] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n13:153678] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n13:153678] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n13:153678] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n13:153678] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n16:148188] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n16:148188] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n16:148188] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n16:148188] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n16:148188] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n16:148188] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n16:148188] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n16:148188] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n16:148188] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n16:148188] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n03:120670] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n03:120670] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n03:120670] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n03:120670] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n03:120670] *** End of error message *** +[b01n11:03986] *** Process received signal *** +[b01n11:03986] Signal: Segmentation fault (11) +[b01n11:03986] Signal code: (3) +[b01n11:03986] Failing at address: 0x14170000000000a0 +[b01n11:03986] [ 0] [0x1000000504d8] +[b01n11:03986] [ 1] [0x3ffff62b4760] +[b01n11:03986] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n11:03986] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n16:08386] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n16:08386] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n16:08386] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n16:08386] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n17:119977] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n17:119977] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n17:119977] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n17:119977] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n18:165696] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n18:165696] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n18:165696] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n18:165696] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n11:134522] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n11:134522] [10] [b22n05:123933] *** Process received signal *** +[0x1000000504d8] +[a28n18:179171] [ 1] [0x3fffea6b1c70] +[a28n18:179171] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n18:179171] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n18:179171] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n18:179171] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n05:139910] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n05:139910] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n05:139910] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n04:44756] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n04:44756] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n03:46688] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n03:46688] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n03:46688] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n03:46688] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n03:46688] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n03:46688] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n03:46688] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n03:46688] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n03:46688] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n03:46688] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n03:46688] [13] [0x3fffe08aff70] +[b12n08:141873] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n08:141873] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n08:141873] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n12:123558] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n12:123558] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n12:123558] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n12:123558] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n05:156009] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n05:156009] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n05:156009] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n05:156009] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n13:123055] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n13:123055] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n13:123055] [ 8] [0x3fffcd4866a0] +[a30n13:154235] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n13:154235] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n13:154235] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n13:154235] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n13:154235] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n13:154235] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n13:154235] [ 8] [b17n13:123517] *** Process received signal *** +[b17n13:123517] Signal: Segmentation fault (11) +[b17n13:123517] Signal code: (3) +[b17n13:123517] Failing at address: 0x14170000000000a0 +[b17n13:123517] [ 0] [0x1000000504d8] +[b17n13:123517] [ 1] [0x3ffff4f727e0] +[b17n13:123517] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n13:123517] [ 3] [b22n10:118386] *** Process received signal *** +[b22n10:118386] Signal: Segmentation fault (11) +[b22n10:118386] Signal code: (3) +[b22n10:118386] Failing at address: 0xb170000000000a0 +[b22n10:118386] [ 0] [0x1000000504d8] +[b22n10:118386] [ 1] [0x3fffc4af0d30] +[b22n10:118386] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n10:118386] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n10:118386] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n10:118386] [ 5] [a30n10:169208] *** Process received signal *** +[a30n10:169208] Signal: Segmentation fault (11) +[a30n10:169208] Signal code: (3) +[a30n10:169208] Failing at address: 0x5170000000000a0 +[a30n10:169208] [ 0] [0x1000000504d8] +[a30n10:169208] [ 1] [0x3fffe10fa770] +[a30n10:169208] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n10:169208] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n10:169208] [ 4] [a29n05:174782] *** Process received signal *** +[a29n05:174782] Signal: Segmentation fault (11) +[a29n05:174782] Signal code: (3) +[a29n05:174782] Failing at address: 0x14170000000000a0 +[a29n05:174782] [ 0] [0x1000000504d8] +[a29n05:174782] [ 1] [0x3fffd6643b60] +[a29n05:174782] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n05:174782] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n05:174782] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n05:174782] [ 5] [a26n16:53798] *** Process received signal *** +[a26n16:53798] Signal: Segmentation fault (11) +[a26n16:53798] Signal code: (3) +[a26n16:53798] Failing at address: 0x14170000000000a0 +[a26n16:53798] [ 0] [0x1000000504d8] +[a26n16:53798] [ 1] [0x3fffd6f8c870] +[a26n16:53798] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n16:53798] [ 3] [b09n08:176680] *** Process received signal *** +[b09n08:176680] Signal: Segmentation fault (11) +[b09n08:176680] Signal code: (3) +[b09n08:176680] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n04:03149] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n04:03149] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n04:03149] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n04:03149] [ 8] [b03n03:06825] *** Process received signal *** +[b03n03:06825] Signal: Segmentation fault (11) +[b03n03:06825] Signal code: (3) +[b03n03:06825] Failing at address: 0x14170000000000a0 +[b03n03:06825] [ 0] [0x1000000504d8] +[b03n03:06825] [ 1] [0x3fffe97715e0] +[b03n03:06825] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n03:06825] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n03:06825] [ 4] [0x1000000504d8] +[b07n12:160964] [ 1] [0x3fffe28e3fb0] +[b07n12:160964] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n12:160964] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n12:160964] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n01:163389] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n01:163389] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n01:163389] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n01:163389] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n01:163389] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n01:163389] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n01:163389] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n01:163389] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n01:163389] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n03:151708] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n03:151708] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n03:151708] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n03:151708] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n03:151708] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n08:116903] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n08:116903] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n08:116903] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n08:116903] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n08:116903] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n08:116903] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n08:116903] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n08:116903] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n08:116903] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n08:116903] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n08:116903] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n08:116903] *** End of error message *** +[0x3fffcb82ed50] +[b11n15:166120] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n15:166120] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n15:166120] [ 4] [a30n07:165007] *** Process received signal *** +[a30n07:165007] Signal: Segmentation fault (11) +[a30n07:165007] Signal code: (3) +[a30n07:165007] Failing at address: 0xb170000000000a0 +[a30n07:165007] [ 0] [0x1000000504d8] +[a30n07:165007] [ 1] [0x3fffc0decfe0] +[a30n07:165007] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n07:165007] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n07:165007] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n07:165007] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n07:165007] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n07:165007] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n07:165007] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n07:165007] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n07:165007] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n07:165007] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n07:165007] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n07:165007] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n07:165007] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n07:165007] *** End of error message *** +[a31n16:162628] *** Process received signal *** +[a31n16:162628] Signal: Segmentation fault (11) +[a31n16:162628] Signal code: (3) +[a31n16:162628] Failing at address: 0x14170000000000a0 +[a31n16:162628] [ 0] [0x1000000504d8] +[a31n16:162628] [ 1] [0x3fffdb5afd00] +[a31n16:162628] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n16:162628] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n06:179452] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n06:179452] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n01:96751] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n01:96751] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n01:96751] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n01:96751] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n01:96751] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n09:12247] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n09:12247] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n09:12247] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n09:12247] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n01:170765] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n01:170765] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n01:170765] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n11:130806] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n11:130806] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n11:130806] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n11:130806] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n11:130806] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n11:130806] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n11:130806] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n11:130806] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n11:130806] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n11:130806] *** End of error message *** +[b13n18:128539] *** Process received signal *** +[b13n18:128539] Signal: Segmentation fault (11) +[b13n18:128539] Signal code: (3) +[b13n18:128539] Failing at address: 0x14170000000000a0 +[b13n18:128539] [ 0] [0x1000000504d8] +[b13n18:128539] [ 1] [0x3fffc8f8c740] +[b13n18:128539] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n18:128539] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n18:123412] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n18:123412] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n18:123412] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n18:123412] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n08:159022] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n08:159022] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n08:159022] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n08:159022] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n08:159022] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n08:159022] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n08:159022] [13] [b15n14:133970] *** Process received signal *** +[b14n08:127445] *** Process received signal *** +[b14n08:127445] Signal: Segmentation fault (11) +[b14n08:127445] Signal code: (3) +[b14n08:127445] Failing at address: 0x14170000000000a0 +[b14n08:127445] [ 0] [0x1000000504d8] +[b14n08:127445] [ 1] [0x3ffff1212c70] +[b14n08:127445] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n08:127445] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n12:145499] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n12:145499] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n12:145499] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n12:145499] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n12:145499] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n12:145499] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n12:145499] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n12:145499] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n12:145499] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n10:140412] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n10:140412] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n10:140412] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n10:140412] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n10:131274] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n10:131274] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n10:131274] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n10:131274] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n07:140136] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n05:04520] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n05:04520] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n16:154184] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n16:154184] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n16:154184] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n16:154184] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n16:154184] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n16:154184] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n03:172648] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n03:172648] [ 5] [a33n11:156154] *** Process received signal *** +[a33n11:156154] Signal: Segmentation fault (11) +[a33n11:156154] Signal code: (3) +[a33n11:156154] Failing at address: 0x14170000000000a0 +[a33n11:156154] [ 0] [0x1000000504d8] +[a33n11:156154] [ 1] [0x3fffe95bade0] +[a33n11:156154] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n11:156154] [ 3] [a32n17:138616] *** Process received signal *** +[a32n17:138616] Signal: Segmentation fault (11) +[a32n17:138616] Signal code: (3) +[a32n17:138616] Failing at address: 0x14170000000000a0 +[a32n17:138616] [ 0] [0x1000000504d8] +[a32n17:138616] [ 1] [0x3fffdaf64060] +[a32n17:138616] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n17:138616] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n05:120230] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n05:120230] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n05:120230] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n05:120230] [ 9] [0x1000000504d8] +[b13n13:133624] [ 1] [0x3fffc1aaade0] +[b13n13:133624] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n13:133624] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n13:133624] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n13:133624] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n13:133624] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n18:149117] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n18:149117] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n18:149117] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n18:149117] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n18:149117] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n18:149117] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n18:149117] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n18:149117] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n18:149117] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n10:126613] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n10:126613] [10] [0x1000000504d8] +[b15n15:122082] [ 1] [0x3ffffbf94ec0] +[b15n15:122082] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n15:122082] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n15:122082] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n15:122082] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n15:122082] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n18:10721] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n18:10721] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n18:10721] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n18:10721] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n18:10721] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n18:10721] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n18:10721] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n18:10721] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n18:10721] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n18:10721] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n18:10721] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n18:10721] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n18:10721] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n10:01250] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n10:01250] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n10:01250] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n10:01250] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n10:01250] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n10:01250] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n10:01250] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n10:01250] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n10:01250] *** End of error message *** +[b17n06:119115] *** Process received signal *** +[b17n06:119115] Signal: Segmentation fault (11) +[b17n06:119115] Signal code: (3) +[b17n06:119115] Failing at address: 0x14170000000000a0 +[b17n06:119115] [ 0] [0x1000000504d8] +[b17n06:119115] [ 1] [0x3ffff4039440] +[b17n06:119115] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n16:166498] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n16:166498] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n16:166498] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n16:166498] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n16:166498] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n16:166498] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n16:166498] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n16:166498] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n16:166498] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n16:166498] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n16:166498] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n16:166498] *** End of error message *** +[b17n01:124278] *** Process received signal *** +[b17n01:124278] Signal: Segmentation fault (11) +[b17n01:124278] Signal code: (3) +[b17n01:124278] Failing at address: 0x14170000000000a0 +[b17n01:124278] [ 0] [0x1000000504d8] +[b17n01:124278] [ 1] [0x3ffff1a8ebd0] +[b17n01:124278] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n01:124278] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n15:120132] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n15:120132] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n10:120976] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n10:120976] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n10:120976] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n08:176176] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n08:176176] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n08:176176] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n08:176176] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n08:176176] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n08:176176] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n08:176176] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n08:176176] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n08:176176] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n08:176176] [13] [0x1000000504d8] +[b13n15:139746] [ 1] [0x3fffce3c8eb0] +[b13n15:139746] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n15:139746] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n15:139746] [ 4] [b06n09:168248] *** Process received signal *** +[b06n09:168248] Signal: Segmentation fault (11) +[b06n09:168248] Signal code: (3) +[b06n09:168248] Failing at address: 0x14170000000000a0 +[b06n09:168248] [ 0] [0x1000000504d8] +[b06n09:168248] [ 1] [0x3fffc6d87750] +[b06n09:168248] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n09:168248] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n09:168248] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n09:168248] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n09:168248] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n09:168248] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n09:168248] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n09:168248] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n09:168248] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n09:168248] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n09:168248] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n09:168248] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n09:168248] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n09:168248] *** End of error message *** +[0x3ffffbb46330] +[b13n14:126464] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n14:126464] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n14:126464] [ 4] [b13n06:133799] *** Process received signal *** +[b13n06:133799] Signal: Segmentation fault (11) +[b13n06:133799] Signal code: (3) +[b13n06:133799] Failing at address: 0xb170000000000a0 +[b13n06:133799] [ 0] [0x1000000504d8] +[b13n06:133799] [ 1] [0x3fffdd89fa50] +[b13n06:133799] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n06:133799] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n06:133799] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n06:133799] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n06:133799] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n06:133799] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n06:133799] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n06:133799] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n06:133799] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n06:133799] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n06:133799] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n06:133799] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n06:133799] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n06:133799] *** End of error message *** +[0x3fffd8613e80] +[a26n14:51450] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n14:51450] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n14:51450] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n14:51450] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n14:51450] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n14:51450] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n14:51450] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n14:51450] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n09:160374] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n09:160374] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n09:160374] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n09:160374] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n09:160374] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n09:160374] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n09:160374] [13] [b09n13:164688] *** Process received signal *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n07:165091] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n07:165091] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n10:10922] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n10:10922] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n10:10922] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n03:09641] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n03:09641] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n03:09641] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n02:122178] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n02:122178] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n02:122178] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n02:122178] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n02:122178] [ 8] [b05n05:173316] *** Process received signal *** +[b05n05:173316] Signal: Segmentation fault (11) +[b05n05:173316] Signal code: (3) +[b05n05:173316] Failing at address: 0xb170000000000a0 +[b05n05:173316] [ 0] [0x1000000504d8] +[b05n05:173316] [ 1] [0x3ffff44f5b30] +[b05n05:173316] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n05:173316] [ 3] [b12n10:140992] *** Process received signal *** +[b12n10:140992] Signal: Segmentation fault (11) +[b12n10:140992] Signal code: (3) +[b12n10:140992] Failing at address: 0xb170000000000a0 +[b12n10:140992] [ 0] [0x1000000504d8] +[b12n10:140992] [ 1] [0x3fffea7dee90] +[b12n10:140992] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n10:140992] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n10:140992] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n10:140992] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n10:140992] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n10:140992] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n10:140992] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n10:140992] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n10:140992] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n10:140992] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n10:140992] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n10:140992] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n10:140992] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n10:140992] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n10:135855] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n10:135855] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n10:135855] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n10:135855] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n14:152748] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n14:152748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n10:169685] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n10:169685] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n10:169685] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n10:169685] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n10:169685] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n10:169685] [b19n01:121742] *** Process received signal *** +[b19n01:121742] Signal: Segmentation fault (11) +[b19n01:121742] Signal code: (3) +[b19n01:121742] Failing at address: 0xb170000000000a0 +[b19n01:121742] [ 0] [0x1000000504d8] +[b19n01:121742] [ 1] [0x3fffd498f0f0] +[b19n01:121742] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n01:121742] [ 3] [a32n07:172877] [ 0] [0x1000000504d8] +[a32n07:172877] [ 1] [0x3fffcdc14170] +[a32n07:172877] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n07:172877] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n07:172877] [ 4] [b10n15:164414] *** Process received signal *** +[b10n15:164414] Signal: Segmentation fault (11) +[b10n15:164414] Signal code: (3) +[b10n15:164414] Failing at address: 0xb170000000000a0 +[b10n15:164414] [ 0] [0x1000000504d8] +[b10n15:164414] [ 1] [0x3fffc6629090] +[b10n15:164414] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n15:164414] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n15:164414] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n15:164414] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n15:164414] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n15:164414] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n15:164414] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n15:164414] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n15:164414] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n15:164414] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n15:164414] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n15:164414] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n15:164414] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n15:164414] *** End of error message *** +[a31n10:155396] *** Process received signal *** +[a31n10:155396] Signal: Segmentation fault (11) +[a31n10:155396] Signal code: (3) +[a31n10:155396] Failing at address: 0xb170000000000a0 +[a31n10:155396] [ 0] [0x1000000504d8] +[a31n10:155396] [ 1] [0x3fffded6f080] +[a31n10:155396] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n10:155396] [ 3] [0x1000000504d8] +[a31n18:169359] [ 1] [0x3fffdf8b3c20] +[a31n18:169359] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n18:169359] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n18:169359] [ 4] [b13n02:125266] *** Process received signal *** +[b13n02:125266] Signal: Segmentation fault (11) +[b13n02:125266] Signal code: (3) +[b13n02:125266] Failing at address: 0x5170000000000a0 +[b13n02:125266] [ 0] [0x1000000504d8] +[b13n02:125266] [ 1] [0x3fffd1c109f0] +[b13n02:125266] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n02:125266] [ 3] [0x3fffc150c890] +[a31n14:165665] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n14:165665] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n14:165665] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n14:165665] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n14:165665] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n14:165665] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n14:165665] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n14:165665] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n14:165665] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n14:165665] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n14:165665] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n14:165665] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n14:165665] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n14:165665] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n04:06832] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n04:06832] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n04:06832] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n04:06832] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n04:06832] *** End of error message *** +[a26n02:74432] *** Process received signal *** +[a26n02:74432] Signal: Segmentation fault (11) +[a26n02:74432] Signal code: (3) +[a26n02:74432] Failing at address: 0x14170000000000a0 +[a26n02:74432] [ 0] [0x1000000504d8] +[a26n02:74432] [ 1] [0x3fffc8533390] +[a26n02:74432] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n02:74432] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n02:74432] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n02:74432] [ 5] [b05n11:176581] *** Process received signal *** +[b05n11:176581] Signal: Segmentation fault (11) +[b05n11:176581] Signal code: (3) +[b05n11:176581] Failing at address: 0x14170000000000a0 +[b05n11:176581] [ 0] [0x1000000504d8] +[b05n11:176581] [ 1] [0x3fffec5a37c0] +[b05n11:176581] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n11:176581] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n11:176581] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n11:176581] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n11:176581] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n11:176581] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n11:176581] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n11:176581] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n11:176581] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n11:176581] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n11:176581] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n11:176581] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n11:176581] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n11:176581] *** End of error message *** +[b20n17:122355] *** Process received signal *** +[b20n17:122355] Signal: Segmentation fault (11) +[b20n17:122355] Signal code: (3) +[b20n17:122355] Failing at address: 0xb170000000000a0 +[b20n17:122355] [ 0] [0x1000000504d8] +[b20n17:122355] [ 1] [0x3ffff4f24580] +[b20n17:122355] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n17:122355] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n17:122355] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n12:160187] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n12:160187] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n12:160187] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n12:160187] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n12:160187] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n12:122872] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n12:122872] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n12:122872] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n12:122872] [ 8] [a31n02:158420] *** Process received signal *** +[a27n07:45306] *** Process received signal *** +[a27n07:45306] Signal: Segmentation fault (11) +[a27n07:45306] Signal code: (3) +[a27n07:45306] Failing at address: 0x14170000000000a0 +[a27n07:45306] [ 0] [0x1000000504d8] +[a27n07:45306] [ 1] [b06n03:164497] *** Process received signal *** +[b06n03:164497] Signal: Segmentation fault (11) +[b06n03:164497] Signal code: (3) +[b06n03:164497] Failing at address: 0x14170000000000a0 +[b06n03:164497] [ 0] [0x1000000504d8] +[b06n03:164497] [ 1] [0x3fffdde0f770] +[b06n03:164497] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n03:164497] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n03:164497] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n03:164497] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n03:164497] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n03:164497] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n03:164497] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n03:164497] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n03:164497] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n03:164497] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n03:164497] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n03:164497] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n03:164497] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n03:164497] *** End of error message *** +[b19n04:130749] [ 0] [0x1000000504d8] +[b19n04:130749] [ 1] [0x3fffcf168170] +[b19n04:130749] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n04:130749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n04:130749] [ 4] [0x1000000504d8] +[a31n13:155851] [ 1] [0x3fffed18c810] +[a31n13:155851] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n13:155851] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n13:155851] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n13:155851] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n13:155851] [ 6] [b07n02:174058] *** Process received signal *** +[b07n02:174058] Signal: Segmentation fault (11) +[b07n02:174058] Signal code: (3) +[b07n02:174058] Failing at address: 0x14170000000000a0 +[b07n02:174058] [ 0] [0x1000000504d8] +[b07n02:174058] [ 1] [0x3fffcb6bd480] +[b07n02:174058] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n02:174058] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n02:174058] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n16:124959] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n16:124959] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n16:124959] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n16:124959] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n16:124959] [14] [b13n04:129406] *** Process received signal *** +[b13n04:129406] Signal: Segmentation fault (11) +[b13n04:129406] Signal code: (3) +[b13n04:129406] Failing at address: 0x14170000000000a0 +[b13n04:129406] [ 0] [0x1000000504d8] +[b13n04:129406] [ 1] [0x3fffe9f381a0] +[b13n04:129406] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n04:129406] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n04:129406] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n04:129406] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n05:149466] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n05:149466] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n05:149466] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n14:177509] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n14:177509] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n14:177509] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n14:177509] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n14:177509] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n14:177509] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n14:177509] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n14:177509] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n14:177509] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n14:177509] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n14:177509] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n14:177509] *** End of error message *** +[a28n07:04681] *** Process received signal *** +[a28n07:04681] Signal: Segmentation fault (11) +[a28n07:04681] Signal code: (3) +[a28n07:04681] Failing at address: 0x14170000000000a0 +[a28n07:04681] [ 0] [0x1000000504d8] +[a28n07:04681] [ 1] [0x3ffffc4699b0] +[a28n07:04681] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n07:04681] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n07:04681] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n07:04681] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n07:04681] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n07:04681] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n07:04681] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n07:04681] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n07:04681] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n07:04681] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n07:04681] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n07:04681] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n07:04681] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n07:04681] *** End of error message *** +[b15n10:122756] *** Process received signal *** +[b15n10:122756] Signal: Segmentation fault (11) +[b15n10:122756] Signal code: (3) +[b15n10:122756] Failing at address: 0x14170000000000a0 +[b15n10:122756] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n09:106790] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n09:106790] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n09:106790] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n09:106790] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n09:106790] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n09:106790] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n09:106790] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n09:106790] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n09:106790] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n09:106790] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n09:170658] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n11:123522] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n11:123522] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n11:123522] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n02:139630] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n02:139630] *** End of error message *** +[a29n04:156414] *** Process received signal *** +[a29n04:156414] Signal: Segmentation fault (11) +[a29n04:156414] Signal code: (3) +[a29n04:156414] Failing at address: 0x14170000000000a0 +[a29n04:156414] [ 0] [0x1000000504d8] +[a29n04:156414] [ 1] [0x3fffcc225570] +[a29n04:156414] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n04:156414] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n04:156414] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n17:163178] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n17:163178] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n17:163178] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n03:135036] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n07:120697] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n07:120697] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n07:120697] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n07:120697] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n07:120697] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n07:120697] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n07:120697] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n09:128494] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n09:128494] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n09:128494] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n09:128494] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n09:128494] [14] [b04n07:02840] *** Process received signal *** +[b04n07:02840] Signal: Segmentation fault (11) +[b04n07:02840] Signal code: (3) +[b04n07:02840] Failing at address: 0x5170000000000a0 +[b07n05:176911] *** Process received signal *** +[b07n05:176911] Signal: Segmentation fault (11) +[b07n05:176911] Signal code: (3) +[b07n05:176911] Failing at address: 0x14170000000000a0 +[b07n05:176911] [ 0] [0x1000000504d8] +[b07n05:176911] [ 1] [0x3fffe8436a10] +[b07n05:176911] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n05:176911] [ 3] [0x1000000504d8] +[b15n08:118554] [ 1] [0x3fffe936bb50] +[b15n08:118554] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n08:118554] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n08:118554] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n04:136998] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n04:136998] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n04:136998] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n04:136998] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n04:136998] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n04:136998] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n04:136998] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n04:136998] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n04:136998] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n04:136998] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n04:136998] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n04:136998] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n08:146119] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n08:146119] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n08:146119] [ 6] [b20n15:132614] *** Process received signal *** +[b20n15:132614] Signal: Segmentation fault (11) +[b20n15:132614] Signal code: (3) +[b20n15:132614] Failing at address: 0xb170000000000a0 +[b20n15:132614] [ 0] [0x1000000504d8] +[b20n15:132614] [ 1] [0x3fffee3c50e0] +[b20n15:132614] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n15:132614] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n15:132614] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n15:132614] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n15:132614] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n15:132614] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n15:132614] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n15:132614] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n15:132614] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n15:132614] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n15:132614] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n15:132614] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n15:132614] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n15:132614] *** End of error message *** +[0x1000000504d8] +[b18n02:125370] [ 1] [0x3ffff7a03360] +[b18n02:125370] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n02:125370] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n02:125370] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n14:126120] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n08:124116] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n08:124116] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n08:124116] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n07:156486] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n07:156486] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n07:156486] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n07:156486] [ 9] [b09n15:165473] *** Process received signal *** +[b09n15:165473] Signal: Segmentation fault (11) +[b09n15:165473] Signal code: (3) +[b09n15:165473] Failing at address: 0x14170000000000a0 +[b09n15:165473] [ 0] [0x1000000504d8] +[b09n15:165473] [ 1] [0x3ffff3cf3110] +[b09n15:165473] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n15:165473] [ 3] [a26n03:68830] *** Process received signal *** +[a26n03:68830] Signal: Segmentation fault (11) +[a26n03:68830] Signal code: (3) +[a26n03:68830] Failing at address: 0xb170000000000a0 +[a26n03:68830] [ 0] [0x1000000504d8] +[a26n03:68830] [ 1] [0x3fffc40cbfe0] +[a26n03:68830] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n03:68830] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n03:68830] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n03:68830] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n03:68830] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n03:68830] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n03:68830] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n03:68830] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n03:68830] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n03:68830] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n03:68830] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n03:68830] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n03:68830] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n03:68830] *** End of error message *** +[a34n13:137745] *** Process received signal *** +[a34n13:137745] Signal: Segmentation fault (11) +[a34n13:137745] Signal code: (3) +[a34n13:137745] Failing at address: 0xb170000000000a0 +[a34n13:137745] [ 0] [0x1000000504d8] +[a34n13:137745] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n07:163253] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n07:163253] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n08:165281] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n08:165281] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n08:165281] [ 5] [a35n12:145263] Signal: Segmentation fault (11) +[a35n12:145263] Signal code: (3) +[a35n12:145263] Failing at address: 0xb170000000000a0 +[a35n12:145263] [ 0] [0x1000000504d8] +[a35n12:145263] [ 1] [0x3fffd4da5e90] +[a35n12:145263] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n12:145263] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n11:134749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n11:134749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n11:134749] [ 5] [0x3fffcfc06e20] +[b12n18:123867] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n18:123867] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n18:123867] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n18:123867] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n18:123867] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n10:176260] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n10:176260] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n10:176260] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n10:176260] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n10:176260] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n10:176260] [11] [b13n03:126186] *** Process received signal *** +[b13n03:126186] Signal: Segmentation fault (11) +[b13n03:126186] Signal code: (3) +[b13n03:126186] Failing at address: 0x14170000000000a0 +[b13n03:126186] [ 0] [0x1000000504d8] +[b13n03:126186] [ 1] [0x3ffff9496b40] +[b13n03:126186] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n03:126186] [ 3] [b18n11:128953] *** Process received signal *** +[b18n11:128953] Signal: Segmentation fault (11) +[b18n11:128953] Signal code: (3) +[b18n11:128953] Failing at address: 0x14170000000000a0 +[b18n11:128953] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n01:123571] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n01:123571] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n01:123571] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n03:119829] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n03:119829] [ 6] [0x3ffff2153660] +[b14n14:122443] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n14:122443] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n14:122443] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n14:122443] [ 5] [a33n03:147536] [ 0] [0x1000000504d8] +[a33n03:147536] [ 1] [0x3fffdb933e30] +[a33n03:147536] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n03:147536] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n03:147536] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n03:147536] [ 5] [b18n07:125450] *** Process received signal *** +[b18n07:125450] Signal: Segmentation fault (11) +[b18n07:125450] Signal code: (3) +[b18n07:125450] Failing at address: 0x14170000000000a0 +[b18n07:125450] [ 0] [b12n16:129236] *** Process received signal *** +[0x3fffce116d20] +[b09n05:173822] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n05:173822] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n05:173822] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n05:173822] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n12:120401] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n12:120401] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n12:120401] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n12:120401] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n04:136610] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n04:136610] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n04:136610] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n04:136610] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n04:136610] [0x1000000504d8] +[a29n13:165387] [ 1] [0x3fffe8174820] +[a29n13:165387] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n13:165387] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n13:165387] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n12:168645] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n12:168645] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n12:168645] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n12:168645] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n12:168645] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n12:168645] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n12:168645] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n12:168645] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n12:168645] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n12:168645] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n12:168645] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n06:148310] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n06:148310] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n03:124680] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n03:124680] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n03:124680] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n03:124680] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n03:124680] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n03:124680] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n03:124680] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n03:124680] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n03:124680] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n03:124680] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n03:124680] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n07:116114] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n07:116114] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n07:116114] *** End of error message *** +[a29n03:155729] *** Process received signal *** +[a29n03:155729] Signal: Segmentation fault (11) +[a29n03:155729] Signal code: (3) +[a29n03:155729] Failing at address: 0xb170000000000a0 +[a29n03:155729] [ 0] [0x1000000504d8] +[a29n03:155729] [ 1] [0x3ffffdd6cb50] +[a29n03:155729] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n03:155729] [ 3] [b11n07:156140] *** Process received signal *** +[b11n07:156140] Signal: Segmentation fault (11) +[b11n07:156140] Signal code: (3) +[b11n07:156140] Failing at address: 0xb170000000000a0 +[b22n14:116410] *** Process received signal *** +[b22n14:116410] Signal: Segmentation fault (11) +[b22n14:116410] Signal code: (3) +[b22n14:116410] Failing at address: 0x14170000000000a0 +[b22n14:116410] [ 0] [0x1000000504d8] +[b22n14:116410] [ 1] [0x3fffc6521a20] +[b03n14:177225] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n14:177225] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n14:177225] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n14:177225] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n14:177225] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n14:177225] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n14:177225] [ 8] [b08n04:172229] *** Process received signal *** +[b08n04:172229] Signal: Segmentation fault (11) +[b08n04:172229] Signal code: (3) +[b08n04:172229] Failing at address: 0xb170000000000a0 +[b08n04:172229] [ 0] [0x1000000504d8] +[b08n04:172229] [ 1] [0x3fffd9148fa0] +[b08n04:172229] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n04:172229] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n04:172229] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n04:172229] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n04:172229] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n04:172229] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n04:172229] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n04:172229] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n04:172229] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n04:172229] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n04:172229] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n04:172229] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n04:172229] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n04:172229] *** End of error message *** +[a27n05:53373] *** Process received signal *** +[a27n05:53373] Signal: Segmentation fault (11) +[a27n05:53373] Signal code: (3) +[a27n05:53373] Failing at address: 0x14170000000000a0 +[a27n05:53373] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n18:121304] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n18:121304] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n18:121304] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n18:121304] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n17:08307] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n17:08307] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n17:08307] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n17:08307] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n17:08307] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n17:08307] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n09:166572] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n09:166572] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n02:01432] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n02:01432] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n04:177599] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n04:177599] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n04:177599] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n04:177599] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n04:177599] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n05:05610] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n05:05610] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n05:05610] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n05:05610] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n05:05610] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n05:05610] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n05:05610] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n05:05610] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n05:05610] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n05:05610] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n05:05610] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n12:177495] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n12:177495] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n12:177495] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n12:177495] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n16:147288] *** End of error message *** +[b13n09:143899] *** Process received signal *** +[b13n09:143899] Signal: Segmentation fault (11) +[b13n09:143899] Signal code: (3) +[b13n09:143899] Failing at address: 0xb170000000000a0 +[b13n09:143899] [ 0] [0x1000000504d8] +[b13n09:143899] [ 1] [0x3ffff740b200] +[b13n09:143899] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n09:143899] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n09:143899] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n09:143899] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n09:143899] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n09:143899] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n09:143899] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n09:143899] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n09:143899] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n09:143899] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n09:143899] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n09:143899] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n09:143899] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n09:143899] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n14:139846] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n14:139846] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n14:139846] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n14:139846] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n14:139846] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n04:117355] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n04:117355] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n04:117355] [ 8] [0x3fffc460e880] +[a26n13:54241] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n13:54241] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n13:54241] [ 4] [0x1000000504d8] +[b20n05:121826] [ 1] [0x3ffffbf5a390] +[b20n05:121826] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n05:121826] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n05:121826] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n01:122015] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n01:122015] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n01:122015] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n01:122015] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n01:122015] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n01:122015] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n01:122015] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n01:122015] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n01:122015] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n01:122015] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n01:122015] *** End of error message *** +[b02n04:168491] *** Process received signal *** +[b02n04:168491] Signal: Segmentation fault (11) +[b02n04:168491] Signal code: (3) +[b02n04:168491] Failing at address: 0x14170000000000a0 +[b02n04:168491] [ 0] [0x1000000504d8] +[b02n04:168491] [ 1] [0x3ffff1c4fe20] +[b02n04:168491] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n04:168491] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n04:168491] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n16:168049] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n16:168049] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n16:168049] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n16:168049] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n07:149478] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n07:149478] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n07:149478] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n07:149478] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n07:149478] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n07:149478] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n07:149478] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n07:149478] *** End of error message *** +[b01n13:177637] *** Process received signal *** +[b01n13:177637] Signal: Segmentation fault (11) +[b01n13:177637] Signal code: (3) +[b01n13:177637] Failing at address: 0x5170000000000a0 +[b01n13:177637] [ 0] [0x1000000504d8] +[b01n13:177637] [ 1] [0x3ffffa872800] +[b01n13:177637] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n13:177637] [ 3] [b02n18:03312] *** Process received signal *** +[b02n18:03312] Signal: Segmentation fault (11) +[b02n18:03312] Signal code: (3) +[b02n18:03312] Failing at address: 0x14170000000000a0 +[b02n18:03312] [ 0] [0x1000000504d8] +[b02n18:03312] [ 1] [a35n13:138091] *** Process received signal *** +[a35n13:138091] Signal: Segmentation fault (11) +[a35n13:138091] Signal code: (3) +[a35n13:138091] Failing at address: 0x5170000000000a0 +[a35n13:138091] [ 0] [0x1000000504d8] +[a35n13:138091] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n08:144853] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n08:144853] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n13:122187] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n13:122187] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n13:122187] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n13:122187] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n13:122187] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n15:32401] [10] [b14n10:124921] *** Process received signal *** +[b14n10:124921] Signal: Segmentation fault (11) +[b14n10:124921] Signal code: (3) +[b14n10:124921] Failing at address: 0x14170000000000a0 +[b14n10:124921] [ 0] [0x1000000504d8] +[b14n10:124921] [ 1] [0x3ffffaaa8170] +[b14n10:124921] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n10:124921] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n10:124921] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n10:124921] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n10:124921] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n10:124921] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n10:124921] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n10:124921] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n10:124921] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n10:124921] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n10:124921] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n10:124921] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n10:124921] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n10:124921] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n16:130375] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n17:125808] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n17:125808] [ 8] [0x1000000504d8] +[a35n08:138174] [ 1] [0x3ffff0e3ab20] +[a35n08:138174] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n08:138174] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n08:138174] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n09:124193] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n09:124193] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n09:124193] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n09:124193] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n09:124193] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n09:124193] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n09:124193] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n09:124193] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n09:124193] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n09:124193] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n10:126193] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n10:126193] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n07:174561] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n07:174561] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n07:174561] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n07:174561] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n07:174561] [10] [b09n07:173044] *** Process received signal *** +[b09n07:173044] Signal: Segmentation fault (11) +[b09n07:173044] Signal code: (3) +[b09n07:173044] Failing at address: 0xb170000000000a0 +[b09n07:173044] [ 0] [0x1000000504d8] +[b09n07:173044] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n12:119806] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n12:119806] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n12:180117] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n12:180117] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n12:180117] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n12:180117] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n14:120639] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n14:120639] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n14:120639] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n14:120639] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n14:120639] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n11:124445] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n11:124445] [ 6] [b19n18:122553] *** Process received signal *** +[b19n18:122553] Signal: Segmentation fault (11) +[b19n18:122553] Signal code: (3) +[b19n18:122553] Failing at address: 0xb170000000000a0 +[b19n18:122553] [ 0] [0x1000000504d8] +[b19n18:122553] [ 1] [0x3fffe0ad12c0] +[b19n18:122553] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n18:122553] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n15:120057] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n15:120057] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n15:120057] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n15:120057] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n14:136137] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n14:136137] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n14:136137] [11] [b05n01:178524] *** Process received signal *** +[b05n01:178524] Signal: Segmentation fault (11) +[b05n01:178524] Signal code: (3) +[b05n01:178524] Failing at address: 0xb170000000000a0 +[b05n01:178524] [ 0] [0x1000000504d8] +[b05n01:178524] [ 1] [0x3fffde0a3800] +[b05n01:178524] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n01:178524] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n10:141233] *** Process received signal *** +[a34n10:141233] Signal: Segmentation fault (11) +[a34n10:141233] Signal code: (3) +[a34n10:141233] Failing at address: 0x14170000000000a0 +[a34n10:141233] [ 0] [0x1000000504d8] +[a34n10:141233] [ 1] [0x3fffeed8d880] +[a34n10:141233] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n10:141233] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n12:164381] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n01:163026] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n01:163026] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n01:163026] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n01:163026] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n01:163026] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n01:163026] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n01:163026] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n01:163026] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n01:163026] [13] [b03n09:173060] *** Process received signal *** +[b03n09:173060] Signal: Segmentation fault (11) +[b03n09:173060] Signal code: (3) +[b03n09:173060] Failing at address: 0xb170000000000a0 +[b03n09:173060] [ 0] [0x1000000504d8] +[b03n09:173060] [ 1] [0x3fffe547bef0] +[b03n09:173060] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n09:173060] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n09:173060] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n09:173060] [ 5] [0x3ffffd00ede0] +[b04n18:176808] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n18:176808] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n18:176808] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n07:08398] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n07:08398] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n07:08398] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n07:08398] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n07:08398] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n07:08398] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n07:08398] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n07:08398] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n07:08398] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n15:148167] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n15:148167] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n15:151252] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n15:151252] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n15:151252] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n18:173979] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n18:173979] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n18:173979] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n18:173979] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n18:173979] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n18:173979] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n18:173979] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n18:173979] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n18:173979] [14] [0x3ffff9a5d650] +[a34n14:150133] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n14:150133] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n14:150133] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n14:150133] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n14:150133] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n02:177548] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n02:177548] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n02:177548] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n15:119251] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n15:119251] *** End of error message *** +[b21n14:119488] *** Process received signal *** +[b21n14:119488] Signal: Segmentation fault (11) +[b21n14:119488] Signal code: (3) +[b21n14:119488] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n13:166925] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n13:166925] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n13:166925] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n13:166925] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n13:166925] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n06:164013] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n06:164013] [ 5] [b08n05:173850] *** Process received signal *** +[b08n05:173850] Signal: Segmentation fault (11) +[b08n05:173850] Signal code: (3) +[b08n05:173850] Failing at address: 0xb170000000000a0 +[b08n05:173850] [ 0] [0x1000000504d8] +[b08n05:173850] [ 1] [0x3fffc139c9a0] +[b08n05:173850] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n05:173850] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n13:117510] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n13:117510] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n13:117510] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n13:117510] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n13:117510] [11] [b23n10:126792] *** Process received signal *** +[b23n10:126792] Signal: Segmentation fault (11) +[b23n10:126792] Signal code: (3) +[b23n10:126792] Failing at address: 0x14170000000000a0 +[b23n10:126792] [ 0] [0x1000000504d8] +[b23n10:126792] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n11:162472] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n11:162472] [ 6] [0x1000000504d8] +[a28n17:168159] [ 1] [0x3fffcf389e00] +[a28n17:168159] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n17:168159] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n17:168159] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n17:168159] [ 5] [0x3fffe9f55af0] +[b22n02:119533] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n02:119533] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n02:119533] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n05:125614] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n05:125614] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n05:125614] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n05:125614] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n18:164797] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n18:164797] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n04:126349] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n04:126349] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n04:126349] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n04:126349] *** End of error message *** +[a35n09:137774] *** Process received signal *** +[a35n09:137774] Signal: Segmentation fault (11) +[a35n09:137774] Signal code: (3) +[a35n09:137774] Failing at address: 0x14170000000000a0 +[a35n09:137774] [ 0] [0x1000000504d8] +[a35n09:137774] [ 1] [0x3fffcc2f2c70] +[a35n09:137774] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n09:137774] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n09:137774] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n09:137774] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n04:179173] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n04:179173] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n04:179173] [ 6] [b15n18:08427] *** Process received signal *** +[b15n18:08427] Signal: Segmentation fault (11) +[b15n18:08427] Signal code: (3) +[b15n18:08427] Failing at address: 0x14170000000000a0 +[b15n18:08427] [ 0] [0x1000000504d8] +[b15n18:08427] [ 1] [0x3ffff66e2190] +[b15n18:08427] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n18:08427] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n11:163259] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n10:134732] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n10:134732] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n04:153965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n04:153965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n04:153965] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n04:153965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n04:153965] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n04:153965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n04:153965] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n04:153965] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n04:153965] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n06:135851] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n06:135851] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n06:135851] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n06:135851] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n06:135851] *** End of error message *** +[a31n03:158655] *** Process received signal *** +[a31n03:158655] Signal: Segmentation fault (11) +[a31n03:158655] Signal code: (3) +[a31n03:158655] Failing at address: 0xb170000000000a0 +[a31n03:158655] [ 0] [a31n11:153415] *** Process received signal *** +[a31n11:153415] Signal: Segmentation fault (11) +[a31n11:153415] Signal code: (3) +[a31n11:153415] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n10:173683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n10:173683] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n18:177222] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n16:173549] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n16:173549] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n16:173549] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n16:173549] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n16:173549] [11] [a33n16:149063] *** Process received signal *** +[a33n16:149063] Signal: Segmentation fault (11) +[a33n16:149063] Signal code: (3) +[a33n16:149063] Failing at address: 0x14170000000000a0 +[a33n16:149063] [ 0] [0x1000000504d8] +[a33n16:149063] [ 1] [0x3ffffe7409b0] +[a33n16:149063] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n16:149063] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n12:178718] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n12:178718] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n12:120130] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n12:120130] [ 8] [0x3ffff536c650] +[a33n13:168162] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n13:168162] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n13:168162] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n13:168162] [ 5] [0x3fffe78a76b0] +[b13n17:129932] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n17:129932] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n17:129932] [ 4] [b21n09:120692] *** Process received signal *** +[b21n09:120692] Signal: Segmentation fault (11) +[b21n09:120692] Signal code: (3) +[b21n09:120692] Failing at address: 0xb170000000000a0 +[b21n09:120692] [ 0] [0x1000000504d8] +[b21n09:120692] [ 1] [0x3fffdc07c250] +[b21n09:120692] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n09:120692] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n09:120692] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n09:120692] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n09:120692] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n09:120692] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n11:124435] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n11:124435] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n11:124435] [ 9] [b11n09:161232] *** Process received signal *** +[b11n09:161232] Signal: Segmentation fault (11) +[b11n09:161232] Signal code: (3) +[b11n09:161232] Failing at address: 0x14170000000000a0 +[b11n09:161232] [ 0] [0x1000000504d8] +[b11n09:161232] [ 1] [0x3ffff1d8a9d0] +[b11n09:161232] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n09:161232] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n15:124803] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n15:124803] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n15:124803] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n15:124803] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n15:124803] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n15:124803] [10] [a34n18:140382] *** Process received signal *** +[a34n18:140382] Signal: Segmentation fault (11) +[a34n18:140382] Signal code: (3) +[a34n18:140382] Failing at address: 0xb170000000000a0 +[a34n18:140382] [ 0] [0x1000000504d8] +[a34n18:140382] [ 1] [0x3ffff8254390] +[a34n18:140382] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n18:140382] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n18:140382] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n18:140382] [ 5] [b21n07:117359] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n07:117359] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n05:164943] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n05:164943] [ 5] [a27n08:54627] *** Process received signal *** +[a27n08:54627] Signal: Segmentation fault (11) +[a27n08:54627] Signal code: (3) +[a27n08:54627] Failing at address: 0x14170000000000a0 +[a27n08:54627] [ 0] [0x1000000504d8] +[a27n08:54627] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n06:127277] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n06:127277] *** End of error message *** +[ 0] [0x1000000504d8] +[a02n04:41670] [ 1] [0x3fffd929d890] +[a02n04:41670] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n04:41670] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n04:41670] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n04:41670] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n04:41670] [ 6] [a32n10:162611] *** Process received signal *** +[a32n10:162611] Signal: Segmentation fault (11) +[a32n10:162611] Signal code: (3) +[a32n10:162611] Failing at address: 0x14170000000000a0 +[a32n10:162611] [ 0] [0x1000000504d8] +[a32n10:162611] [ 1] [0x3fffed7bbe40] +[a32n10:162611] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n10:162611] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n04:166208] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n04:166208] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n04:166208] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n04:166208] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n04:166208] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n04:166208] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n04:166208] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n04:166208] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n04:166208] [13] [b08n17:171199] *** Process received signal *** +[b08n17:171199] Signal: Segmentation fault (11) +[b08n17:171199] Signal code: (3) +[b08n17:171199] Failing at address: 0x5170000000000a0 +[b08n17:171199] [ 0] [0x1000000504d8] +[b08n17:171199] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n09:127999] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n09:127999] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n09:127999] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n09:127999] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n08:116172] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n08:116172] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n08:116172] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n08:116172] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n16:165920] [ 4] [a27n04:52707] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n13:123888] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n13:123888] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n13:123888] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n13:123888] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n08:131378] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n08:131378] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n08:131378] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n08:131378] [10] [b13n07:133373] *** Process received signal *** +[b13n07:133373] Signal: Segmentation fault (11) +[b13n07:133373] Signal code: (3) +[b13n07:133373] Failing at address: 0xb170000000000a0 +[b13n07:133373] [ 0] [0x1000000504d8] +[b13n07:133373] [ 1] [0x3ffffa707f00] +[b13n07:133373] [ 2] [b05n02:176050] *** Process received signal *** +[b05n02:176050] Signal: Segmentation fault (11) +[b05n02:176050] Signal code: (3) +[b05n02:176050] Failing at address: 0x5170000000000a0 +[b05n02:176050] [ 0] [0x1000000504d8] +[b05n02:176050] [ 1] [0x3fffcf6c32e0] +[b05n02:176050] [b14n18:131585] *** Process received signal *** +[b14n18:131585] Signal: Segmentation fault (11) +[b14n18:131585] Signal code: (3) +[b14n18:131585] Failing at address: 0xb170000000000a0 +[b14n18:131585] [ 0] [0x1000000504d8] +[b14n18:131585] [ 1] [0x3fffc53a5ff0] +[b14n18:131585] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n18:131585] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n18:131585] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n18:131585] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n18:131585] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n16:128443] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n16:128443] *** End of error message *** +[0x3fffe8ecc300] +[b17n16:127353] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n16:127353] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n16:127353] [ 4] [a33n02:167598] *** Process received signal *** +[a33n02:167598] Signal: Segmentation fault (11) +[a33n02:167598] Signal code: (3) +[a33n02:167598] Failing at address: 0x14170000000000a0 +[a33n02:167598] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n14:133062] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n14:133062] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n14:133062] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n14:133062] [ 8] [b16n18:134184] *** Process received signal *** +[b16n18:134184] Signal: Segmentation fault (11) +[b16n18:134184] Signal code: (3) +[b16n18:134184] Failing at address: 0xb170000000000a0 +[b16n18:134184] [ 0] [0x1000000504d8] +[b16n18:134184] [ 1] [0x3ffff1fc67b0] +[b16n18:134184] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n18:134184] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n18:134184] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n18:134184] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n18:134184] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n18:134184] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n18:134184] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n18:134184] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n18:134184] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n18:134184] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n18:134184] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n18:134184] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n18:134184] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n18:134184] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n17:124167] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n17:124167] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n17:124167] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n17:124167] [ 9] [a29n18:07422] *** Process received signal *** +[a29n18:07422] Signal: Segmentation fault (11) +[a29n18:07422] Signal code: (3) +[a29n18:07422] Failing at address: 0x14170000000000a0 +[a29n18:07422] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n13:122032] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n13:122032] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n13:122032] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n13:122032] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n13:122032] [ 8] [0x1000000504d8] +[a26n08:76223] [ 1] [0x3ffff5ed4e50] +[a26n08:76223] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n08:76223] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n08:76223] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n08:76223] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n10:124502] *** End of error message *** +[0x1000000504d8] +[b15n12:132100] [ 1] [0x3fffd9668000] +[b15n12:132100] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n12:132100] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n12:132100] [ 4] [0x1000000504d8] +[a31n06:167368] [ 1] [0x3fffef2064c0] +[a31n06:167368] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n06:167368] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n06:167368] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n18:69155] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n18:69155] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n18:69155] [ 8] [b07n14:176070] *** Process received signal *** +[b07n14:176070] Signal: Segmentation fault (11) +[b07n14:176070] Signal code: (3) +[b07n14:176070] Failing at address: 0x14170000000000a0 +[b07n14:176070] [ 0] [0x1000000504d8] +[b07n14:176070] [ 1] [0x3fffeee57f90] +[b07n14:176070] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n14:176070] [ 3] [a27n11:23061] *** Process received signal *** +[0x1000000504d8] +[a02n18:21771] [ 1] [0x3fffdcc30870] +[a02n18:21771] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n18:21771] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n18:21771] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n18:21771] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n18:21771] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n18:21771] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n18:21771] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n18:21771] [ 9] [b07n08:172900] *** Process received signal *** +[b07n08:172900] Signal: Segmentation fault (11) +[b07n08:172900] Signal code: (3) +[b07n08:172900] Failing at address: 0xb170000000000a0 +[b07n08:172900] [ 0] [0x1000000504d8] +[b07n08:172900] [ 1] [0x3fffeb78fd50] +[b07n08:172900] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n08:172900] [ 3] [a34n09:149099] *** Process received signal *** +[a34n09:149099] Signal: Segmentation fault (11) +[a34n09:149099] Signal code: (3) +[a34n09:149099] Failing at address: 0xb170000000000a0 +[a34n09:149099] [ 0] [0x1000000504d8] +[a34n09:149099] [ 1] [0x3fffee788250] +[a34n09:149099] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n09:149099] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n01:48805] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n01:48805] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n01:48805] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n01:48805] [ 8] [0x1000000504d8] +[b20n01:128829] [ 1] [0x3fffd6d37a10] +[b20n01:128829] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n01:128829] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n01:128829] [ 4] [b07n11:170898] *** Process received signal *** +[b07n11:170898] Signal: Segmentation fault (11) +[b07n11:170898] Signal code: (3) +[b07n11:170898] Failing at address: 0x14170000000000a0 +[b07n11:170898] [ 0] [0x1000000504d8] +[b07n11:170898] [ 1] [0x3fffe9c0d280] +[b07n11:170898] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n11:170898] [ 3] [b07n01:166961] *** Process received signal *** +[b07n01:166961] Signal: Segmentation fault (11) +[b07n01:166961] Signal code: (3) +[b07n01:166961] Failing at address: 0xb170000000000a0 +[b07n01:166961] [ 0] [0x1000000504d8] +[b07n01:166961] [ 1] [0x3ffffafa2aa0] +[b07n01:166961] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n01:166961] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n01:166961] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n01:166961] [ 5] [b19n11:133970] *** Process received signal *** +[b19n11:133970] Signal: Segmentation fault (11) +[b19n11:133970] Signal code: (3) +[b19n11:133970] Failing at address: 0xb170000000000a0 +[b19n11:133970] [ 0] [b02n07:180193] *** Process received signal *** +[b02n07:180193] Signal: Segmentation fault (11) +[b02n07:180193] Signal code: (3) +[b02n07:180193] Failing at address: 0x14170000000000a0 +[b02n07:180193] [ 0] [0x1000000504d8] +[b02n07:180193] [ 1] [0x3fffcd70c010] +[b02n07:180193] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n07:180193] [ 3] [ 0] [0x1000000504d8] +[a27n09:51331] [ 1] [0x3fffe2ce90a0] +[a27n09:51331] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n09:51331] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n09:51331] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n03:38501] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n03:38501] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n03:123619] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n03:123619] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n03:123619] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n03:123619] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n03:123619] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n14:08951] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n16:116526] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n16:116526] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n16:116526] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n16:116526] [ 9] [a34n16:135792] *** Process received signal *** +[a34n16:135792] Signal: Segmentation fault (11) +[a34n16:135792] Signal code: (3) +[a34n16:135792] Failing at address: 0x14170000000000a0 +[a34n16:135792] [ 0] [0x1000000504d8] +[a34n16:135792] [ 1] [0x3fffc6d93350] +[a34n16:135792] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n16:135792] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n16:135792] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n16:135792] [ 5] [b10n14:168375] *** Process received signal *** +[b10n14:168375] Signal: Segmentation fault (11) +[b10n14:168375] Signal code: (3) +[b10n14:168375] Failing at address: 0xb170000000000a0 +[b10n14:168375] [ 0] [0x1000000504d8] +[b10n14:168375] [ 1] [0x3fffec6de620] +[b10n14:168375] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n14:168375] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n12:12761] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n12:12761] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n12:12761] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n12:12761] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n12:12761] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n12:12761] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n12:12761] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n12:167678] [ 6] [b10n11:160348] *** Process received signal *** +[b10n11:160348] Signal: Segmentation fault (11) +[b10n11:160348] Signal code: (3) +[b10n11:160348] Failing at address: 0x14170000000000a0 +[b10n11:160348] [ 0] [0x1000000504d8] +[b10n11:160348] [ 1] [0x3fffdd4e69a0] +[b10n11:160348] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n11:160348] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n07:118167] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n07:118167] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n07:118167] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n07:118167] [ 9] [0x3ffffa483040] +[a29n15:177058] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n15:177058] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n15:177058] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n15:177058] [ 5] [b01n17:179115] *** Process received signal *** +[b01n17:179115] Signal: Segmentation fault (11) +[b01n17:179115] Signal code: (3) +[b01n17:179115] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n10:142929] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n10:142929] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n10:142929] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n10:142929] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n10:142929] [13] [0x3fffffca6ee0] +[a31n15:155432] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n15:155432] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n15:155432] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n15:155432] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n15:155432] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n15:155432] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n15:155432] [ 8] [0x1000000504d8] +[b14n13:122144] [ 1] [0x3fffc81be9b0] +[b14n13:122144] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n13:122144] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n13:122144] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n13:114398] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n13:114398] [ 5] [0x3ffff40628b0] +[b13n05:135165] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n05:135165] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n05:135165] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n05:135165] [ 5] [a31n05:158573] *** Process received signal *** +[a31n05:158573] Signal: Segmentation fault (11) +[a31n05:158573] Signal code: (3) +[a31n05:158573] Failing at address: 0xb170000000000a0 +[a31n05:158573] [ 0] [0x1000000504d8] +[a31n05:158573] [ 1] [0x3fffdb052df0] +[a31n05:158573] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n05:158573] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n05:158573] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n05:158573] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n05:158573] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n05:158573] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n05:158573] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n05:158573] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n05:158573] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n05:158573] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n05:158573] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n05:158573] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n05:158573] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n05:158573] *** End of error message *** +[b22n08:118582] *** Process received signal *** +[b22n08:118582] Signal: Segmentation fault (11) +[b22n08:118582] Signal code: (3) +[b22n08:118582] Failing at address: 0x14170000000000a0 +[b22n08:118582] [ 0] [0x1000000504d8] +[b22n08:118582] [ 1] [0x3ffffa94d740] +[b22n08:118582] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n08:118582] [ 3] [b19n09:123419] *** Process received signal *** +[b19n09:123419] Signal: Segmentation fault (11) +[b19n09:123419] Signal code: (3) +[b19n09:123419] Failing at address: 0x14170000000000a0 +[b19n09:123419] [ 0] [0x1000000504d8] +[b19n09:123419] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n16:157889] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n16:157889] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n06:126374] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n06:126374] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n06:126374] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n06:126374] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n06:126374] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n14:128871] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n14:128871] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n14:128871] [ 6] [b02n01:167298] *** Process received signal *** +[b02n01:167298] Signal: Segmentation fault (11) +[b02n01:167298] Signal code: (3) +[b02n01:167298] Failing at address: 0xb170000000000a0 +[b02n01:167298] [ 0] [0x1000000504d8] +[b02n01:167298] [ 1] [b02n17:03844] *** Process received signal *** +[b02n17:03844] Signal: Segmentation fault (11) +[b02n17:03844] Signal code: (3) +[b02n17:03844] Failing at address: 0xb170000000000a0 +[b02n17:03844] [ 0] [0x1000000504d8] +[b02n17:03844] [ 1] [0x3fffda7e1070] +[b02n17:03844] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n17:03844] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n17:03844] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n17:03844] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n17:03844] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n17:03844] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n17:03844] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n17:03844] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n17:03844] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n17:03844] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n17:03844] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n17:03844] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n17:03844] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n17:03844] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n18:179699] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n18:179699] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n18:179699] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n18:179699] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n18:179699] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n02:06561] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n02:06561] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n02:06561] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n02:06561] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n17:149904] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n17:149904] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n14:133632] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n14:133632] [ 7] [a28n09:03927] *** Process received signal *** +[a28n09:03927] Signal: Segmentation fault (11) +[a28n09:03927] Signal code: (3) +[a28n09:03927] Failing at address: 0x14170000000000a0 +[a28n09:03927] [ 0] [a28n13:176367] *** Process received signal *** +[a28n13:176367] Signal: Segmentation fault (11) +[a28n13:176367] Signal code: (3) +[a28n13:176367] Failing at address: 0x14170000000000a0 +[a28n13:176367] [ 0] [0x1000000504d8] +[a28n13:176367] [ 1] [0x3fffe52c0dc0] +[a28n13:176367] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n13:176367] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n14:169522] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n14:169522] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n14:169522] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n10:176311] *** End of error message *** +[b22n12:122964] *** Process received signal *** +[b22n12:122964] Signal: Segmentation fault (11) +[b22n12:122964] Signal code: (3) +[b22n12:122964] Failing at address: 0x14170000000000a0 +[b22n12:122964] [ 0] [0x1000000504d8] +[b22n12:122964] [ 1] [0x3fffc5c9c540] +[b22n12:122964] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n12:122964] [ 3] [0x3fffe5424350] +[b02n06:43019] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n06:43019] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n06:43019] [ 4] [a33n01:141818] *** Process received signal *** +[a33n01:141818] Signal: Segmentation fault (11) +[a33n01:141818] Signal code: (3) +[a33n01:141818] Failing at address: 0x14170000000000a0 +[a33n01:141818] [ 0] [0x1000000504d8] +[a33n01:141818] [ 1] [0x3ffffa51a1b0] +[a33n01:141818] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n06:01841] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n06:01841] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n06:01841] [ 6] [a35n14:136662] *** Process received signal *** +[a35n14:136662] Signal: Segmentation fault (11) +[a35n14:136662] Signal code: (3) +[a35n14:136662] Failing at address: 0x5170000000000a0 +[a35n14:136662] [ 0] [0x3ffffdf98420] +[a35n07:138201] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n07:138201] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n07:138201] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n07:138201] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n03:177927] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n03:177927] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n14:175715] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n14:175715] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n14:175715] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n14:175715] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n14:175715] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n14:175715] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n14:175715] [10] [b02n02:166747] *** Process received signal *** +[b02n02:166747] Signal: Segmentation fault (11) +[b02n02:166747] Signal code: (3) +[b02n02:166747] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n14:159239] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n14:159239] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n14:159239] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n08:08957] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n06:143011] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n06:143011] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n06:143011] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n06:143011] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n06:143011] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n16:124665] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n16:124665] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n16:124665] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n16:124665] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n17:15873] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n17:15873] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n17:15873] [ 5] [a33n08:147822] *** Process received signal *** +[a33n08:147822] Signal: Segmentation fault (11) +[a33n08:147822] Signal code: (3) +[a33n08:147822] Failing at address: 0x14170000000000a0 +[a33n08:147822] [ 0] [0x1000000504d8] +[a33n08:147822] [ 1] [0x3fffffc85030] +[a33n08:147822] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n08:147822] [ 3] [b17n18:126903] *** Process received signal *** +[b17n18:126903] Signal: Segmentation fault (11) +[b17n18:126903] Signal code: (3) +[b17n18:126903] Failing at address: 0x14170000000000a0 +[b17n18:126903] [ 0] [0x1000000504d8] +[b17n18:126903] [ 1] [0x3ffff22fe830] +[b17n18:126903] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n18:126903] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n17:138142] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n17:138142] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n07:132090] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n07:132090] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n07:132090] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n07:132090] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n13:156920] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n13:156920] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n13:156920] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n13:156920] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n10:67858] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n10:67858] [ 6] [b10n02:164489] *** Process received signal *** +[b10n02:164489] Signal: Segmentation fault (11) +[b10n02:164489] Signal code: (3) +[b10n02:164489] Failing at address: 0xb170000000000a0 +[b10n02:164489] [ 0] [0x1000000504d8] +[b10n02:164489] [ 1] [0x3ffff53765d0] +[b10n02:164489] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n02:164489] [ 3] [a33n04:169913] *** Process received signal *** +[a33n04:169913] Signal: Segmentation fault (11) +[a33n04:169913] Signal code: (3) +[a33n04:169913] Failing at address: 0x14170000000000a0 +[a33n04:169913] [ 0] [0x1000000504d8] +[a33n04:169913] [ 1] [0x3fffc3fc3840] +[a33n04:169913] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n07:153177] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n07:153177] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n07:153177] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n07:153177] [ 9] [0x1000000504d8] +[b19n07:131671] [ 1] [0x3ffff2465960] +[b19n07:131671] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n07:131671] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n07:131671] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n07:131671] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n07:131671] [ 6] [0x1000000504d8] +[b15n13:123708] [ 1] [0x3fffeb04d3e0] +[b15n13:123708] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n13:123708] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n13:123708] [ 4] [a29n02:163025] *** Process received signal *** +[a29n02:163025] Signal: Segmentation fault (11) +[a29n02:163025] Signal code: (3) +[a29n02:163025] Failing at address: 0xb170000000000a0 +[a29n02:163025] [ 0] [0x1000000504d8] +[a29n02:163025] [ 1] [0x3ffff5d06e60] +[a29n02:163025] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n02:163025] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n02:163025] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n02:163025] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n02:163025] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n02:163025] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n02:163025] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n02:163025] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n02:163025] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n02:163025] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n02:163025] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n02:163025] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n02:163025] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n02:163025] *** End of error message *** +[a31n17:156182] *** Process received signal *** +[a31n17:156182] Signal: Segmentation fault (11) +[a31n17:156182] Signal code: (3) +[a31n17:156182] Failing at address: 0xb170000000000a0 +[a31n17:156182] [ 0] [0x1000000504d8] +[a31n17:156182] [ 1] [0x3fffe36af7d0] +[a31n17:156182] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n17:156182] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n16:01571] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n16:01571] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n16:01571] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n11:152988] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n11:152988] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n11:152988] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n11:152988] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n11:152988] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n11:152988] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n11:152988] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n11:01170] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n11:01170] [13] [b12n03:139012] *** Process received signal *** +[b12n03:139012] Signal: Segmentation fault (11) +[b12n03:139012] Signal code: (3) +[b12n03:139012] Failing at address: 0x14170000000000a0 +[b12n03:139012] [ 0] [0x1000000504d8] +[b12n03:139012] [ 1] [0x3ffffe1f2400] +[b12n03:139012] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n03:139012] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n03:139012] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n03:139012] [ 5] [b01n09:173749] *** Process received signal *** +[b01n09:173749] Signal: Segmentation fault (11) +[b01n09:173749] Signal code: (3) +[b01n09:173749] Failing at address: 0xb170000000000a0 +[b01n09:173749] [ 0] [0x1000000504d8] +[b01n09:173749] [ 1] [0x3ffff0bc60a0] +[b01n09:173749] [ 2] [a27n18:28216] *** Process received signal *** +[a27n18:28216] Signal: Segmentation fault (11) +[a27n18:28216] Signal code: (3) +[a27n18:28216] Failing at address: 0x14170000000000a0 +[a27n18:28216] [ 0] [b12n05:138194] *** Process received signal *** +[b12n05:138194] Signal: Segmentation fault (11) +[b12n05:138194] Signal code: (3) +[b12n05:138194] Failing at address: 0x5170000000000a0 +[b12n05:138194] [ 0] [0x1000000504d8] +[b12n05:138194] [ 1] [0x3fffc28e7d60] +[b12n05:138194] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n05:138194] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n05:138194] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n05:138194] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n05:138194] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n05:138194] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n05:138194] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n05:138194] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n05:138194] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n05:138194] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n05:138194] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n05:138194] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n05:138194] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n05:138194] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n01:128157] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n01:128157] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n15:85939] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n15:85939] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n15:85939] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n15:85939] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n15:85939] [10] [b11n17:164293] *** Process received signal *** +[b11n17:164293] Signal: Segmentation fault (11) +[b11n17:164293] Signal code: (3) +[b11n17:164293] Failing at address: 0x14170000000000a0 +[b11n17:164293] [ 0] [0x1000000504d8] +[b11n17:164293] [ 1] [0x3fffeb425390] +[b11n17:164293] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n17:164293] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n17:164293] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n17:164293] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n09:121754] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n09:121754] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n11:137873] [ 5] [ 0] [0x1000000504d8] +[a36n17:150927] [ 1] [0x3fffd99bfec0] +[a36n17:150927] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n17:150927] [ 3] [b09n11:171965] *** Process received signal *** +[b09n11:171965] Signal: Segmentation fault (11) +[b09n11:171965] Signal code: (3) +[b09n11:171965] Failing at address: 0xb170000000000a0 +[b09n11:171965] [ 0] [0x1000000504d8] +[b09n11:171965] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n15:176426] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n15:176426] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n15:176426] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n15:176426] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n15:176426] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n15:176426] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n15:176426] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n15:176426] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n15:176426] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n15:176426] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n15:176426] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n15:176426] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n03:10291] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n03:10291] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n03:10291] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n08:124726] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n08:124726] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n08:124726] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n08:124726] [10] [b12n02:143968] *** Process received signal *** +[b12n02:143968] Signal: Segmentation fault (11) +[b12n02:143968] Signal code: (3) +[b12n02:143968] Failing at address: 0xb170000000000a0 +[b12n02:143968] [ 0] [0x1000000504d8] +[b12n02:143968] [ 1] [0x3fffd51bd790] +[b12n02:143968] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n02:143968] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n13:01300] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n13:01300] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n10:130583] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n10:130583] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n10:130583] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n10:130583] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n10:130583] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n10:130583] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n16:153293] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n16:153293] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n16:153293] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n16:153293] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n16:153293] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n16:153293] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n06:122164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n06:122164] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n06:122164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n06:122164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n12:11141] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n12:11141] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n12:11141] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n12:11141] [14] [b11n14:161617] *** Process received signal *** +[b11n14:161617] Signal: Segmentation fault (11) +[b11n14:161617] Signal code: (3) +[b11n14:161617] Failing at address: 0xb170000000000a0 +[b11n14:161617] [ 0] [0x1000000504d8] +[b11n14:161617] [ 1] [0x3fffdef070f0] +[b11n14:161617] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n14:161617] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n13:158865] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n13:158865] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n13:158865] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n13:158865] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n13:158865] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n13:158865] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n13:158865] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n13:158865] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n13:158865] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n13:158865] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n13:158865] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n13:158865] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n11:152843] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n11:152843] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n11:152843] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n11:152843] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n11:152843] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n11:152843] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n08:174412] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n08:174412] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n08:174412] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n08:174412] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n08:174412] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n08:174412] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n08:174412] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n08:174412] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n01:130410] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n01:130410] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n01:130410] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n01:130410] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n01:130410] [ 9] [b18n09:123812] *** Process received signal *** +[b18n09:123812] Signal: Segmentation fault (11) +[b18n09:123812] Signal code: (3) +[b18n09:123812] Failing at address: 0x14170000000000a0 +[b18n09:123812] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n16:171919] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n16:171919] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n16:171919] [11] [0x1000000504d8] +[b04n15:177861] [ 1] [0x3ffffdc64460] +[b04n15:177861] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n15:177861] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n15:177861] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n15:177861] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n15:177861] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n15:177861] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n15:177861] [ 8] [0x3fffff4b09d0] +[b14n16:122494] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n16:122494] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n16:122494] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n16:122494] [ 5] [b17n12:123234] *** Process received signal *** +[b17n12:123234] Signal: Segmentation fault (11) +[b17n12:123234] Signal code: (3) +[b17n12:123234] Failing at address: 0xb170000000000a0 +[b17n12:123234] [ 0] [0x1000000504d8] +[b17n12:123234] [ 1] [0x3fffd2eccd40] +[b17n12:123234] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n12:123234] [ 3] [a31n01:173779] *** Process received signal *** +[a31n01:173779] Signal: Segmentation fault (11) +[a31n01:173779] Signal code: (3) +[a31n01:173779] Failing at address: 0xb170000000000a0 +[a31n01:173779] [ 0] [0x1000000504d8] +[a31n01:173779] [ 1] [0x3ffff2acd8d0] +[a31n01:173779] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n01:173779] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n01:173779] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n01:173779] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n01:173779] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n01:173779] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n01:173779] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n03:127630] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n03:127630] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n03:127630] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n03:127630] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n09:119549] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n09:119549] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n09:119549] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n10:151892] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n10:151892] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n10:151892] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n17:119583] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n17:119583] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n17:119583] [ 5] [b12n01:145733] *** Process received signal *** +[b12n01:145733] Signal: Segmentation fault (11) +[b12n01:145733] Signal code: (3) +[b12n01:145733] Failing at address: 0x14170000000000a0 +[b12n01:145733] [ 0] [0x1000000504d8] +[b12n01:145733] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n15:148580] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n15:148580] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n15:148580] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n10:122210] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n10:122210] *** End of error message *** +[a36n18:163106] [ 0] [0x1000000504d8] +[a36n18:163106] [ 1] [0x3ffff9826760] +[a36n18:163106] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n18:163106] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n18:163106] [ 4] [b12n04:138159] *** Process received signal *** +[b12n04:138159] Signal: Segmentation fault (11) +[b12n04:138159] Signal code: (3) +[b12n04:138159] Failing at address: 0xb170000000000a0 +[b12n04:138159] [ 0] [0x1000000504d8] +[b12n04:138159] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n04:176000] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n04:176000] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n04:176000] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n12:148383] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n12:148383] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n12:148383] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n12:148383] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n12:52454] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n12:52454] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n12:52454] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n12:52454] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n12:52454] [ 9] [0x3fffd9a79360] +[b10n17:164559] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n17:164559] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n17:164559] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n17:164559] [ 5] [b11n18:163573] *** Process received signal *** +[b11n18:163573] Signal: Segmentation fault (11) +[b11n18:163573] Signal code: (3) +[b11n18:163573] Failing at address: 0x14170000000000a0 +[b11n18:163573] [ 0] [0x1000000504d8] +[b11n18:163573] [ 1] [0x3fffcd551d70] +[b11n18:163573] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n18:163573] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n03:168210] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n03:168210] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n03:168210] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n03:168210] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n02:120190] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n02:120190] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n02:120190] [ 6] [0x3fffd11b5d90] +[b06n02:163321] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n02:163321] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n02:163321] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n02:163321] [ 5] [0x3fffe81ac160] +[b06n14:173895] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n14:173895] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n14:173895] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n14:173895] [ 5] [0x1000000504d8] +[a34n08:163160] [ 1] [0x3ffff2785c70] +[a34n08:163160] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n08:163160] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n08:163160] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n08:163160] [ 5] [a30n09:152061] *** Process received signal *** +[a30n09:152061] Signal: Segmentation fault (11) +[a30n09:152061] Signal code: (3) +[a30n09:152061] Failing at address: 0x14170000000000a0 +[a30n09:152061] [ 0] [0x1000000504d8] +[a30n09:152061] [ 1] [0x3fffc0a04980] +[a30n09:152061] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n09:152061] [ 3] [b22n09:122069] *** Process received signal *** +[b22n09:122069] Signal: Segmentation fault (11) +[b22n09:122069] Signal code: (3) +[b22n09:122069] Failing at address: 0xb170000000000a0 +[b22n09:122069] [ 0] [0x1000000504d8] +[b22n09:122069] [ 1] [0x3fffcf5e01a0] +[b22n09:122069] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n09:122069] [ 3] [ 0] [0x1000000504d8] +[a31n08:156371] [ 1] [0x3fffe6acc6f0] +[a31n08:156371] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n08:156371] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n08:156371] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n08:156371] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n11:176299] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n11:176299] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n07:127354] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n07:127354] [ 5] [b09n06:167093] *** Process received signal *** +[b09n06:167093] Signal: Segmentation fault (11) +[b09n06:167093] Signal code: (3) +[b09n06:167093] Failing at address: 0xb170000000000a0 +[b09n06:167093] [ 0] [0x1000000504d8] +[b09n06:167093] [ 1] [0x3ffff83d4100] +[b09n06:167093] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n06:167093] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n06:167093] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n06:167093] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n06:167093] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n06:167093] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n06:167093] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n06:167093] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n06:167093] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n06:167093] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n06:167093] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n06:167093] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n06:167093] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n06:167093] *** End of error message *** +[0x3fffdf30f940] +[a31n12:156917] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n12:156917] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n12:156917] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n12:156917] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n12:156917] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n06:155088] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n06:155088] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n06:155088] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n06:155088] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n06:155088] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n06:155088] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n06:155088] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n07:125472] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n07:125472] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n07:125472] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n07:125472] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n07:125472] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n07:125472] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n08:129888] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n08:129888] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n08:129888] [ 6] [0x3fffea543230] +[a34n05:148377] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n05:148377] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n05:148377] [ 4] [a32n13:163195] *** Process received signal *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n11:147712] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n11:147712] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n13:165263] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n11:175598] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n11:175598] *** End of error message *** +[b07n04:164047] *** Process received signal *** +[b07n04:164047] Signal: Segmentation fault (11) +[b07n04:164047] Signal code: (3) +[b07n04:164047] Failing at address: 0xb170000000000a0 +[b07n04:164047] [ 0] [0x1000000504d8] +[b07n04:164047] [ 1] [0x3ffff0fd8ad0] +[b07n04:164047] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n04:164047] [ 3] [b07n15:166703] *** Process received signal *** +[b07n15:166703] Signal: Segmentation fault (11) +[b07n15:166703] Signal code: (3) +[b07n15:166703] Failing at address: 0xb170000000000a0 +[b07n15:166703] [ 0] [0x1000000504d8] +[b07n15:166703] [ 1] [0x3fffeef6a120] +[b07n15:166703] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n15:166703] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n15:166703] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n15:166703] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n15:166703] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n15:166703] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n15:166703] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n15:166703] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n15:166703] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n15:166703] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n15:166703] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n15:166703] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n15:166703] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n15:166703] *** End of error message *** +[0x3fffd48d6390] +[b01n16:166689] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n16:166689] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n16:166689] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n16:166689] [ 5] [a27n06:60149] *** Process received signal *** +[a27n06:60149] Signal: Segmentation fault (11) +[a27n06:60149] Signal code: (3) +[a27n06:60149] Failing at address: 0xb170000000000a0 +[a27n06:60149] [ 0] [0x1000000504d8] +[a27n06:60149] [ 1] [0x3fffda437b50] +[a27n06:60149] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n06:60149] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n06:60149] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n06:60149] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n06:60149] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n06:60149] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n06:60149] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n06:60149] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n06:60149] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n06:60149] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n06:60149] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n06:60149] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n06:60149] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n06:60149] *** End of error message *** +[a30n02:155379] *** Process received signal *** +[a30n02:155379] Signal: Segmentation fault (11) +[a30n02:155379] Signal code: (3) +[a30n02:155379] Failing at address: 0xb170000000000a0 +[a30n02:155379] [ 0] [0x1000000504d8] +[a30n02:155379] [ 1] [0x3fffe56b5610] +[a30n02:155379] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n02:155379] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n02:155379] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n08:163101] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n08:163101] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n08:163101] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n08:163101] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n08:163101] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n08:163101] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n08:163101] [13] [b06n01:165165] *** Process received signal *** +[b06n01:165165] Signal: Segmentation fault (11) +[b06n01:165165] Signal code: (3) +[b06n01:165165] Failing at address: 0x5170000000000a0 +[b06n01:165165] [ 0] [0x1000000504d8] +[b06n01:165165] [ 1] [0x3ffffdf689a0] +[b06n01:165165] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n01:165165] [ 3] [0x1000000504d8] +[a30n15:154385] [ 1] [0x3fffe7f36fe0] +[a30n15:154385] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n15:154385] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n15:154385] [ 4] [a35n18:145743] *** Process received signal *** +[a35n18:145743] Signal: Segmentation fault (11) +[a35n18:145743] Signal code: (3) +[a35n18:145743] Failing at address: 0x14170000000000a0 +[a35n18:145743] [ 0] [0x1000000504d8] +[a35n18:145743] [ 1] [0x3fffd1c8edd0] +[a35n18:145743] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n18:145743] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n18:145743] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n18:145743] [ 5] [b01n15:03733] [ 0] [0x1000000504d8] +[b01n15:03733] [ 1] [0x3fffc6d452c0] +[b01n15:03733] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n15:03733] [ 3] [b10n04:169412] *** Process received signal *** +[b10n04:169412] Signal: Segmentation fault (11) +[b10n04:169412] Signal code: (3) +[b10n04:169412] Failing at address: 0xb170000000000a0 +[b10n04:169412] [ 0] [0x1000000504d8] +[b10n04:169412] [ 1] [0x3fffe55cda10] +[b10n04:169412] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n04:169412] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n04:169412] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n04:169412] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n04:169412] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n04:169412] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n04:169412] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n04:169412] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n04:169412] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n04:169412] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n04:169412] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n04:169412] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n04:169412] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n04:169412] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n15:137008] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n15:137008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n15:137008] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n12:177118] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n12:177118] *** End of error message *** +[a02n06:38119] *** Process received signal *** +[a02n06:38119] Signal: Segmentation fault (11) +[a02n06:38119] Signal code: (3) +[a02n06:38119] Failing at address: 0x14170000000000a0 +[a02n06:38119] [ 0] [0x1000000504d8] +[a02n06:38119] [ 1] [0x3fffcbfc98f0] +[a02n06:38119] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n05:169787] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n05:169787] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n05:169787] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n05:169787] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n05:169787] *** End of error message *** +[a29n09:79006] *** Process received signal *** +[a29n09:79006] Signal: Segmentation fault (11) +[a29n09:79006] Signal code: (3) +[a29n09:79006] Failing at address: 0xb170000000000a0 +[a29n09:79006] [ 0] [0x1000000504d8] +[a29n09:79006] [ 1] [0x3fffe96ddd40] +[a29n09:79006] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n09:79006] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n12:154342] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n12:154342] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n06:177848] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n06:177848] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n06:177848] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n06:177848] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n08:121687] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n08:121687] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n08:121687] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n08:121687] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n08:121687] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n03:39688] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n03:39688] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n03:39688] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n03:39688] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n03:39688] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n03:39688] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n03:39688] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n03:39688] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n03:39688] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n03:39688] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n03:39688] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n11:120241] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n11:120241] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n11:120241] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n06:10224] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n06:10224] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n06:10224] [a35n02:149227] *** Process received signal *** +[a35n02:149227] Signal: Segmentation fault (11) +[a35n02:149227] Signal code: (3) +[a35n02:149227] Failing at address: 0x14170000000000a0 +[a35n02:149227] [ 0] [0x1000000504d8] +[a35n02:149227] [ 1] [0x3ffff64e99c0] +[a35n02:149227] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n12:115636] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n12:115636] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n12:115636] [ 8] [a35n01:138004] *** Process received signal *** +[a35n01:138004] Signal: Segmentation fault (11) +[a35n01:138004] Signal code: (3) +[a35n01:138004] Failing at address: 0xb170000000000a0 +[a35n01:138004] [ 0] [0x1000000504d8] +[a35n01:138004] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n15:123041] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n15:123041] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n15:123041] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n15:123041] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n15:123041] [ 8] [a27n02:49108] *** Process received signal *** +[a27n02:49108] Signal: Segmentation fault (11) +[a27n02:49108] Signal code: (3) +[a27n02:49108] Failing at address: 0x14170000000000a0 +[a27n02:49108] [ 0] [0x1000000504d8] +[a27n02:49108] [ 1] [0x3ffffdea91e0] +[a27n02:49108] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n02:49108] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n02:49108] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n02:49108] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n02:49108] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n02:49108] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n02:49108] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n02:49108] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n02:49108] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n02:49108] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n02:49108] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n02:49108] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n02:49108] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n02:49108] *** End of error message *** +[a32n11:152814] [ 0] [0x1000000504d8] +[a32n11:152814] [ 1] [0x3fffe56290c0] +[a32n11:152814] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n11:152814] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n11:152814] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n11:152814] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n11:152814] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n11:152814] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n11:152814] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n11:152814] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n11:152814] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n11:152814] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n11:152814] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n11:152814] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n11:152814] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n11:152814] *** End of error message *** +[0x1000000504d8] +[b02n11:02684] [ 1] [0x3fffe8cbf180] +[b02n11:02684] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n11:02684] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n11:02684] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n17:146798] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n17:146798] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n17:146798] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n17:146798] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n12:140120] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n12:140120] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n12:140120] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n03:168929] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n03:168929] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n03:168929] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n03:168929] [ 8] [a31n09:164364] *** Process received signal *** +[a31n09:164364] Signal: Segmentation fault (11) +[a31n09:164364] Signal code: (3) +[a31n09:164364] Failing at address: 0xb170000000000a0 +[a31n09:164364] [ 0] [0x1000000504d8] +[a31n09:164364] [ 1] [0x3fffd41b8700] +[a31n09:164364] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n09:164364] [ 3] [a35n03:135163] *** Process received signal *** +[a35n03:135163] Signal: Segmentation fault (11) +[a35n03:135163] Signal code: (3) +[a35n03:135163] Failing at address: 0xb170000000000a0 +[a35n03:135163] [ 0] [0x1000000504d8] +[a35n03:135163] [ 1] [0x3ffffd19ca40] +[a35n03:135163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n03:135163] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n03:135163] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n03:135163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n03:135163] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n03:135163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n03:135163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n03:135163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n03:135163] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n03:135163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n03:135163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n03:135163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n03:135163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n03:135163] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n17:119467] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n17:119467] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n12:11420] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n12:11420] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n12:11420] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n12:11420] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n17:126526] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n17:126526] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n17:126526] [ 8] [a30n14:163550] *** Process received signal *** +[a30n14:163550] Signal: Segmentation fault (11) +[a30n14:163550] Signal code: (3) +[a30n14:163550] Failing at address: 0x14170000000000a0 +[a30n14:163550] [ 0] [0x1000000504d8] +[a30n14:163550] [ 1] [0x3fffc8a1fb50] +[a30n14:163550] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n14:163550] [ 3] [b06n15:161757] *** Process received signal *** +[b06n15:161757] Signal: Segmentation fault (11) +[b06n15:161757] Signal code: (3) +[b06n15:161757] Failing at address: 0x14170000000000a0 +[b06n15:161757] [ 0] [0x1000000504d8] +[b06n15:161757] [ 1] [a29n01:153683] *** Process received signal *** +[a29n01:153683] Signal: Segmentation fault (11) +[a29n01:153683] Signal code: (3) +[a29n01:153683] Failing at address: 0xb170000000000a0 +[a29n01:153683] [ 0] [0x1000000504d8] +[a29n01:153683] [ 1] [0x3fffc7e0c4f0] +[a29n01:153683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n01:153683] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n16:124797] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n16:124797] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n16:124797] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n16:124797] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n16:124797] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n16:124797] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n16:124797] [10] [b07n06:177004] *** Process received signal *** +[b07n06:177004] Signal: Segmentation fault (11) +[b07n06:177004] Signal code: (3) +[b07n06:177004] Failing at address: 0x5170000000000a0 +[b07n06:177004] [ 0] [0x1000000504d8] +[b07n06:177004] [ 1] [a27n14:21317] *** Process received signal *** +[a27n14:21317] Signal: Segmentation fault (11) +[a27n14:21317] Signal code: (3) +[a27n14:21317] Failing at address: 0x14170000000000a0 +[a27n14:21317] [ 0] [0x1000000504d8] +[a27n14:21317] [ 1] [0x3ffff4b9d600] +[a27n14:21317] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n14:21317] [ 3] [a33n09:147790] *** Process received signal *** +[a33n09:147790] Signal: Segmentation fault (11) +[a33n09:147790] Signal code: (3) +[a33n09:147790] Failing at address: 0xb170000000000a0 +[a33n09:147790] [ 0] [0x1000000504d8] +[a33n09:147790] [ 1] [a30n03:167777] *** Process received signal *** +[a30n03:167777] Signal: Segmentation fault (11) +[a30n03:167777] Signal code: (3) +[a30n03:167777] Failing at address: 0x5170000000000a0 +[a30n03:167777] [ 0] [0x1000000504d8] +[a30n03:167777] [ 1] [0x3fffc83ef190] +[a30n03:167777] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n03:167777] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n03:167777] [ 4] [b18n03:136668] *** Process received signal *** +[b18n03:136668] Signal: Segmentation fault (11) +[b18n03:136668] Signal code: (3) +[b18n03:136668] Failing at address: 0xb170000000000a0 +[b18n03:136668] [ 0] [0x1000000504d8] +[b18n03:136668] [ 1] [0x3fffd9b70000] +[b18n03:136668] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n03:136668] [ 3] [b23n03:125241] *** Process received signal *** +[b23n03:125241] Signal: Segmentation fault (11) +[b23n03:125241] Signal code: (3) +[b23n03:125241] Failing at address: 0x14170000000000a0 +[b23n03:125241] [ 0] [0x1000000504d8] +[b23n03:125241] [ 1] [0x3ffffe0ef030] +[b23n03:125241] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n03:125241] [ 3] [a26n01:65726] *** Process received signal *** +[a26n01:65726] Signal: Segmentation fault (11) +[a26n01:65726] Signal code: (3) +[a26n01:65726] Failing at address: 0xb170000000000a0 +[a26n01:65726] [ 0] [0x1000000504d8] +[a26n01:65726] [ 1] [0x3fffe0b71d40] +[a26n01:65726] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n01:65726] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n06:154622] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n06:154622] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n06:154622] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n06:154622] [13] [b06n05:163964] *** Process received signal *** +[b06n05:163964] Signal: Segmentation fault (11) +[b06n05:163964] Signal code: (3) +[b06n05:163964] Failing at address: 0xb170000000000a0 +[b06n05:163964] [ 0] [0x1000000504d8] +[b06n05:163964] [ 1] [0x3fffe937c980] +[b06n05:163964] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n05:163964] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n05:163964] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n05:163964] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n05:163964] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n05:163964] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n05:163964] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n05:163964] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n05:163964] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n05:163964] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n05:163964] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n05:163964] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n05:163964] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n05:163964] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n05:162565] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n05:162565] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n05:162565] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n03:01352] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n03:01352] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n03:01352] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n04:169707] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n04:169707] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n04:169707] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n04:169707] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n04:169707] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n04:169707] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n04:169707] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n04:169707] [13] [a35n11:148675] *** Process received signal *** +[a35n11:148675] Signal: Segmentation fault (11) +[a35n11:148675] Signal code: (3) +[a35n11:148675] Failing at address: 0x5170000000000a0 +[a35n11:148675] [ 0] [0x1000000504d8] +[a35n11:148675] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n04:118344] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n04:118344] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n04:118344] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n02:127220] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n02:127220] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n02:127220] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n02:127220] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n02:127220] [11] [a26n04:54378] *** Process received signal *** +[a26n04:54378] Signal: Segmentation fault (11) +[a26n04:54378] Signal code: (3) +[a26n04:54378] Failing at address: 0xb170000000000a0 +[a26n04:54378] [ 0] [b20n09:121727] *** Process received signal *** +[b20n09:121727] Signal: Segmentation fault (11) +[b20n09:121727] Signal code: (3) +[b20n09:121727] Failing at address: 0xb170000000000a0 +[b20n09:121727] [ 0] [0x1000000504d8] +[b20n09:121727] [ 1] [0x3fffd406e1b0] +[b20n09:121727] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n09:121727] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n09:121727] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n09:121727] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n09:121727] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n09:121727] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n09:121727] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n09:121727] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n09:121727] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n09:121727] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n09:121727] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n09:121727] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n09:121727] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n09:121727] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n08:172427] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n08:172427] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n08:172427] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n08:172427] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n17:163817] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n17:163817] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n17:163817] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n17:163817] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n17:163817] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n17:163817] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n17:163817] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n17:163817] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n17:163817] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n09:174690] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n09:174690] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n09:174690] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n09:174690] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n09:174690] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n09:174690] *** End of error message *** +[a28n02:178475] *** Process received signal *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n02:122560] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n07:03459] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n07:03459] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n07:03459] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n10:161720] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n10:161720] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n10:161720] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n10:161720] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n10:161720] *** End of error message *** +[b07n09:163517] *** Process received signal *** +[b07n09:163517] Signal: Segmentation fault (11) +[b07n09:163517] Signal code: (3) +[b07n09:163517] Failing at address: 0x14170000000000a0 +[b07n09:163517] [ 0] [0x1000000504d8] +[b07n09:163517] [ 1] [0x3fffcacebb00] +[b07n09:163517] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n09:163517] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n09:163517] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n09:163517] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n09:163517] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n09:163517] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n09:163517] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n09:163517] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n09:163517] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n09:163517] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n09:163517] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n09:163517] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n09:163517] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n09:163517] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n17:162361] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n17:162361] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n01:164963] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n06:130544] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n06:130544] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n02:123707] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n02:123707] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n02:123707] [ 6] [b18n06:133953] *** Process received signal *** +[b18n06:133953] Signal: Segmentation fault (11) +[b18n06:133953] Signal code: (3) +[b18n06:133953] Failing at address: 0xb170000000000a0 +[b18n06:133953] [ 0] [ 0] [0x1000000504d8] +[a28n04:13081] [ 1] [0x3ffff17cc780] +[a28n04:13081] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n04:13081] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n04:13081] [ 4] [0x1000000504d8] +[b19n05:131261] [ 1] [0x3ffff7395ba0] +[b19n05:131261] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n03:151881] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n03:151881] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n03:151881] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n03:151881] [12] [a28n01:179263] *** Process received signal *** +[a28n01:179263] Signal: Segmentation fault (11) +[a28n01:179263] Signal code: (3) +[a28n01:179263] Failing at address: 0x14170000000000a0 +[a28n01:179263] [ 0] [0x1000000504d8] +[a28n01:179263] [ 1] [b18n05:135123] *** Process received signal *** +[b18n05:135123] Signal: Segmentation fault (11) +[b18n05:135123] Signal code: (3) +[b18n05:135123] Failing at address: 0xb170000000000a0 +[b18n05:135123] [ 0] [0x1000000504d8] +[b18n05:135123] [ 1] [0x3fffc1469bd0] +[b18n05:135123] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n05:135123] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n08:175807] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n08:175807] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n08:175807] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n08:175807] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n08:175807] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n08:175807] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n08:175807] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n08:175807] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n08:175807] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n08:175807] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n08:175807] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n08:175807] *** End of error message *** +[a26n05:54535] [ 0] [0x1000000504d8] +[a26n05:54535] [ 1] [0x3fffe7f884f0] +[a26n05:54535] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n05:54535] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n05:54535] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n16:121928] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n16:121928] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n16:121928] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n16:121928] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n11:164997] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n11:164997] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n11:164997] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n11:164997] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n11:164997] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n11:164997] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n11:164997] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n11:164997] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n04:142256] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n04:142256] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n04:142256] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n04:142256] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n03:153854] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n03:153854] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n03:153854] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n03:153854] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n03:153854] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n03:153854] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n05:130805] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n05:130805] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n05:130805] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n05:130805] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n05:130805] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n05:130805] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n05:130805] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n05:130805] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n05:130805] [13] [b21n18:118680] [ 0] [0x1000000504d8] +[b21n18:118680] [ 1] [0x3fffca02d270] +[b21n18:118680] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n18:118680] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n01:171787] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n01:171787] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n01:171787] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n01:171787] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n01:171787] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n01:171787] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n01:171787] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n10:164940] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n10:164940] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n10:164940] [ 6] [b21n17:118575] *** Process received signal *** +[b21n17:118575] Signal: Segmentation fault (11) +[b21n17:118575] Signal code: (3) +[b21n17:118575] Failing at address: 0x14170000000000a0 +[b21n17:118575] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n06:162919] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n06:162919] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n06:162919] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n06:162919] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n06:162919] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n06:162919] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n06:162919] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n06:162919] [14] [a36n14:152040] *** Process received signal *** +[a36n14:152040] Signal: Segmentation fault (11) +[a36n14:152040] Signal code: (3) +[a36n14:152040] Failing at address: 0x14170000000000a0 +[a36n14:152040] [ 0] [0x1000000504d8] +[a36n14:152040] [ 1] [0x3fffe65aa7d0] +[a36n14:152040] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n14:152040] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n14:152040] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n14:152040] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n14:152040] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n14:152040] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n14:152040] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n14:152040] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n14:152040] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n14:152040] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n14:152040] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n14:152040] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n14:152040] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n14:152040] *** End of error message *** +[a36n01:141341] *** Process received signal *** +[a36n01:141341] Signal: Segmentation fault (11) +[a36n01:141341] Signal code: (3) +[a36n01:141341] Failing at address: 0xb170000000000a0 +[a36n01:141341] [ 0] [0x1000000504d8] +[a36n01:141341] [ 1] [0x3fffd90eb510] +[a36n01:141341] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n01:141341] [ 3] [a36n02:147885] *** Process received signal *** +[a36n02:147885] Signal: Segmentation fault (11) +[a36n02:147885] Signal code: (3) +[a36n02:147885] Failing at address: 0x5170000000000a0 +[a36n02:147885] [ 0] [0x1000000504d8] +[a36n02:147885] [ 1] [0x3fffde7423f0] +[a36n02:147885] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n02:147885] [ 3] [a32n05:154186] *** Process received signal *** +[a32n05:154186] Signal: Segmentation fault (11) +[a32n05:154186] Signal code: (3) +[a32n05:154186] Failing at address: 0xb170000000000a0 +[a32n05:154186] [ 0] [0x1000000504d8] +[a32n05:154186] [ 1] [0x3fffecd890c0] +[a32n05:154186] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n05:154186] [ 3] [a32n01:158305] *** Process received signal *** +[a32n01:158305] Signal: Segmentation fault (11) +[a32n01:158305] Signal code: (3) +[a32n01:158305] Failing at address: 0x14170000000000a0 +[a32n01:158305] [ 0] [0x1000000504d8] +[a32n01:158305] [ 1] [0x3fffcd7f44e0] +[a32n01:158305] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n01:158305] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n01:158305] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n01:158305] [ 5] [b16n02:121682] *** Process received signal *** +[b16n02:121682] Signal: Segmentation fault (11) +[b16n02:121682] Signal code: (3) +[b16n02:121682] Failing at address: 0x14170000000000a0 +[b16n02:121682] [ 0] [0x1000000504d8] +[b16n02:121682] [ 1] [0x3fffe555bb80] +[b16n02:121682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n02:121682] [ 3] [a28n16:171168] *** Process received signal *** +[a28n16:171168] Signal: Segmentation fault (11) +[a28n16:171168] Signal code: (3) +[a28n16:171168] Failing at address: 0x14170000000000a0 +[a28n16:171168] [ 0] [0x1000000504d8] +[a28n16:171168] [ 1] [0x3fffdc457930] +[a28n16:171168] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n16:171168] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n16:171168] [ 4] [b01n06:171504] *** Process received signal *** +[b01n06:171504] Signal: Segmentation fault (11) +[b01n06:171504] Signal code: (3) +[b01n06:171504] Failing at address: 0x5170000000000a0 +[b01n06:171504] [ 0] [0x1000000504d8] +[b01n06:171504] [ 1] [0x3fffe1d9b530] +[b01n06:171504] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n06:171504] [ 3] [b16n04:123055] *** Process received signal *** +[b16n04:123055] Signal: Segmentation fault (11) +[b16n04:123055] Signal code: (3) +[b16n04:123055] Failing at address: 0x14170000000000a0 +[b16n04:123055] [ 0] [0x1000000504d8] +[b16n04:123055] [ 1] [0x3fffc70f44b0] +[b16n04:123055] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n04:123055] [ 3] [b10n18:162422] *** Process received signal *** +[b10n18:162422] Signal: Segmentation fault (11) +[b10n18:162422] Signal code: (3) +[b10n18:162422] Failing at address: 0xb170000000000a0 +[b10n18:162422] [ 0] [0x1000000504d8] +[b10n18:162422] [ 1] [0x3ffff5e2c440] +[b10n18:162422] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n18:162422] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n18:162422] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n18:162422] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n18:162422] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n18:162422] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n18:162422] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n18:162422] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n18:162422] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n18:162422] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n18:162422] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n18:162422] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n18:162422] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n18:162422] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n08:164751] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n08:164751] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n08:164751] [12] [b22n06:133053] *** Process received signal *** +[b22n06:133053] Signal: Segmentation fault (11) +[b22n06:133053] Signal code: (3) +[b22n06:133053] Failing at address: 0xb170000000000a0 +[b22n06:133053] [ 0] [0x1000000504d8] +[b22n06:133053] [ 1] [0x3fffc0a044d0] +[b22n06:133053] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n06:133053] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n06:133053] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n06:133053] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n06:133053] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n06:133053] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n06:133053] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n06:133053] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n06:133053] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n06:133053] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n06:133053] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n06:133053] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n06:133053] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n06:133053] *** End of error message *** +[b16n03:121810] *** Process received signal *** +[b16n03:121810] Signal: Segmentation fault (11) +[b16n03:121810] Signal code: (3) +[b16n03:121810] Failing at address: 0xb170000000000a0 +[b16n03:121810] [ 0] [0x1000000504d8] +[b16n03:121810] [ 1] [0x3ffff6d5be30] +[b16n03:121810] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n03:121810] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n03:121810] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n03:121810] [ 5] [b16n01:132062] *** Process received signal *** +[b16n01:132062] Signal: Segmentation fault (11) +[b16n01:132062] Signal code: (3) +[b16n01:132062] Failing at address: 0x14170000000000a0 +[b16n01:132062] [ 0] [0x1000000504d8] +[b16n01:132062] [ 1] [0x3ffff0048e20] +[b16n01:132062] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n01:132062] [ 3] [b03n07:03371] *** Process received signal *** +[b03n07:03371] Signal: Segmentation fault (11) +[b03n07:03371] Signal code: (3) +[b03n07:03371] Failing at address: 0xb170000000000a0 +[b03n04:04849] *** Process received signal *** +[b03n04:04849] Signal: Segmentation fault (11) +[b03n04:04849] Signal code: (3) +[b03n04:04849] Failing at address: 0xb170000000000a0 +[b03n04:04849] [ 0] [0x1000000504d8] +[b03n04:04849] [ 1] [0x3fffee2afe90] +[b03n04:04849] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n04:04849] [ 3] [b16n06:132107] *** Process received signal *** +[b16n06:132107] Signal: Segmentation fault (11) +[b16n06:132107] Signal code: (3) +[b16n06:132107] Failing at address: 0xb170000000000a0 +[b16n06:132107] [ 0] [0x1000000504d8] +[b16n06:132107] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n04:133905] *** End of error message *** +[b23n02:116886] *** Process received signal *** +[b23n02:116886] Signal: Segmentation fault (11) +[b23n02:116886] Signal code: (3) +[b23n02:116886] Failing at address: 0xb170000000000a0 +[b23n02:116886] [ 0] [0x1000000504d8] +[b23n02:116886] [ 1] [0x3fffe279e100] +[b23n02:116886] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n02:116886] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n02:116886] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n02:116886] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n02:116886] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n02:116886] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n02:116886] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n02:116886] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n02:116886] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n02:116886] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n02:116886] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n02:116886] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n02:116886] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n02:116886] *** End of error message *** +[b01n01:146656] *** Process received signal *** +[b01n01:146656] Signal: Segmentation fault (11) +[b01n01:146656] Signal code: (3) +[b01n01:146656] Failing at address: 0xb170000000000a0 +[b01n01:146656] [ 0] [0x1000000504d8] +[b01n01:146656] [ 1] [0x3fffcdf0df40] +[b01n01:146656] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n01:146656] [ 3] [b01n05:02908] *** Process received signal *** +[b01n05:02908] Signal: Segmentation fault (11) +[b01n05:02908] Signal code: (3) +[b01n05:02908] Failing at address: 0x14170000000000a0 +[b01n05:02908] [ 0] [0x1000000504d8] +[b01n05:02908] [ 1] [0x3fffebae4a20] +[b01n05:02908] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n05:02908] [ 3] [b16n07:119636] *** Process received signal *** +[b16n07:119636] Signal: Segmentation fault (11) +[b16n07:119636] Signal code: (3) +[b16n07:119636] Failing at address: 0xb170000000000a0 +[b16n07:119636] [ 0] [0x1000000504d8] +[b16n07:119636] [ 1] [0x3fffed29f200] +[b16n07:119636] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n07:119636] [ 3] [a36n09:140252] *** Process received signal *** +[a36n09:140252] Signal: Segmentation fault (11) +[a36n09:140252] Signal code: (3) +[a36n09:140252] Failing at address: 0xb170000000000a0 +[a36n09:140252] [ 0] [0x1000000504d8] +[a36n09:140252] [ 1] [0x3fffdcdce920] +[a36n09:140252] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n09:140252] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n09:140252] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n09:140252] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n09:140252] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n09:140252] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n09:140252] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n09:140252] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n09:140252] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n09:140252] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n09:140252] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n09:140252] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n09:140252] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n09:140252] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n06:120735] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n06:120735] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n06:120735] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n06:120735] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n06:120735] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n06:120735] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n06:120735] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n06:120735] [12] [b03n10:02712] *** Process received signal *** +[b03n10:02712] Signal: Segmentation fault (11) +[b03n10:02712] Signal code: (3) +[b03n10:02712] Failing at address: 0x14170000000000a0 +[b03n10:02712] [ 0] [0x1000000504d8] +[b03n10:02712] [ 1] [0x3fffe613f5e0] +[b03n10:02712] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n10:02712] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n10:02712] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n10:02712] [ 5] [b16n08:121433] *** Process received signal *** +[b16n08:121433] Signal: Segmentation fault (11) +[b16n08:121433] Signal code: (3) +[b16n08:121433] Failing at address: 0x14170000000000a0 +[b16n08:121433] [ 0] [0x1000000504d8] +[b16n08:121433] [ 1] [0x3fffe2b9dea0] +[b16n08:121433] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n08:121433] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n08:121433] [ 4] [b02n15:169493] *** Process received signal *** +[b02n15:169493] Signal: Segmentation fault (11) +[b02n15:169493] Signal code: (3) +[b02n15:169493] Failing at address: 0x14170000000000a0 +[b02n15:169493] [ 0] [0x1000000504d8] +[b02n15:169493] [ 1] [0x3fffc8637130] +[b02n15:169493] [ 2] [ 0] [0x1000000504d8] +[b15n07:124352] [ 1] [0x3fffcd36e610] +[b15n07:124352] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n07:124352] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n07:124352] [ 4] [0x3fffea6f9e70] +[b06n06:163678] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n06:163678] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n06:163678] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n06:163678] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n06:163678] [ 6] [b14n03:130345] *** Process received signal *** +[b14n03:130345] Signal: Segmentation fault (11) +[b14n03:130345] Signal code: (3) +[b14n03:130345] Failing at address: 0xb170000000000a0 +[b14n03:130345] [ 0] [0x1000000504d8] +[b14n03:130345] [ 1] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n02:139023] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n02:139023] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n02:139023] [ 5] [b06n11:160798] *** Process received signal *** +[b06n11:160798] Signal: Segmentation fault (11) +[b06n11:160798] Signal code: (3) +[b06n11:160798] Failing at address: 0x14170000000000a0 +[b06n11:160798] [ 0] [0x1000000504d8] +[b06n11:160798] [ 1] [0x3ffff8353e90] +[b06n11:160798] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n11:160798] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n05:173082] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n05:173082] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n05:173082] *** End of error message *** +[a33n06:165301] *** Process received signal *** +[a33n06:165301] Signal: Segmentation fault (11) +[a33n06:165301] Signal code: (3) +[a33n06:165301] Failing at address: 0x5170000000000a0 +[a33n06:165301] [ 0] [0x1000000504d8] +[a33n06:165301] [ 1] [0x3fffdc0bb800] +[a33n06:165301] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n06:165301] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n06:165301] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n06:165301] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n06:165301] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n06:165301] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n06:165301] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n06:165301] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n06:165301] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n06:165301] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n06:165301] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n06:165301] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n06:165301] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n06:165301] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n12:138738] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n12:138738] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n12:138738] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n12:138738] [10] [b15n05:120333] *** Process received signal *** +[b15n05:120333] Signal: Segmentation fault (11) +[b15n05:120333] Signal code: (3) +[b15n05:120333] Failing at address: 0x14170000000000a0 +[b15n05:120333] [ 0] [0x1000000504d8] +[b15n05:120333] [ 1] [0x3fffe56e8f40] +[b15n05:120333] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n05:120333] [ 3] [b10n16:164549] *** Process received signal *** +[b10n16:164549] Signal: Segmentation fault (11) +[b10n16:164549] Signal code: (3) +[b10n16:164549] Failing at address: 0xb170000000000a0 +[b10n16:164549] [ 0] [0x1000000504d8] +[b10n16:164549] [ 1] [b14n04:126860] [ 0] [0x1000000504d8] +[b14n04:126860] [ 1] [0x3fffdd1fe4f0] +[b14n04:126860] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n04:126860] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n04:126860] [ 4] [b16n05:122272] *** Process received signal *** +[b16n05:122272] Signal: Segmentation fault (11) +[b16n05:122272] Signal code: (3) +[b16n05:122272] Failing at address: 0x14170000000000a0 +[b16n05:122272] [ 0] [0x1000000504d8] +[b16n05:122272] [ 1] [0x3fffce4f38c0] +[b16n05:122272] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n05:122272] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n05:122272] [ 4] [a32n15:139057] *** Process received signal *** +[a32n15:139057] Signal: Segmentation fault (11) +[a32n15:139057] Signal code: (3) +[a32n15:139057] Failing at address: 0x14170000000000a0 +[a32n15:139057] [ 0] [0x1000000504d8] +[a32n15:139057] [ 1] [0x3ffffd9ee4d0] +[a32n15:139057] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n15:139057] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n15:139057] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n15:139057] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n15:139057] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n15:139057] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n15:139057] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n15:139057] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n15:139057] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n15:139057] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n15:139057] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n15:139057] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n15:139057] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n15:139057] *** End of error message *** +[0x1000000504d8] +[a33n17:168151] [ 1] [0x3fffed7f0690] +[a33n17:168151] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n17:168151] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n17:168151] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n17:168151] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n09:155756] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n09:155756] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n09:155756] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n09:155756] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n17:176413] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n17:176413] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n17:176413] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n17:176413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n17:176413] [ 8] [b19n17:123235] *** Process received signal *** +[b19n17:123235] Signal: Segmentation fault (11) +[b19n17:123235] Signal code: (3) +[b19n17:123235] Failing at address: 0xb170000000000a0 +[b19n17:123235] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n03:154296] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n03:154296] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n03:154296] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n03:154296] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n03:154296] [ 9] [0x3fffc2a11900] +[b14n01:128004] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n01:128004] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n01:128004] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n01:165127] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n01:165127] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n01:165127] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n01:165127] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n01:165127] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n01:165127] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n01:165127] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n06:157791] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n06:157791] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n06:157791] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n06:157791] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n06:157791] *** End of error message *** +[a35n06:138765] Signal: Segmentation fault (11) +[a35n06:138765] Signal code: (3) +[a35n06:138765] Failing at address: 0x14170000000000a0 +[a35n06:138765] [ 0] [0x1000000504d8] +[a35n06:138765] [ 1] [0x3fffc213d180] +[a35n06:138765] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n06:138765] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n06:138765] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n06:138765] [ 5] [b08n18:164008] *** Process received signal *** +[b08n18:164008] Signal: Segmentation fault (11) +[b08n18:164008] Signal code: (3) +[b08n18:164008] Failing at address: 0xb170000000000a0 +[b08n18:164008] [ 0] [0x1000000504d8] +[b08n18:164008] [ 1] [0x3fffd4415a60] +[b08n18:164008] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n18:164008] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n18:164008] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n18:164008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n18:164008] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n18:164008] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n18:164008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n18:164008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n18:164008] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n18:164008] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n18:164008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n18:164008] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n18:164008] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n18:164008] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n11:117476] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n11:117476] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n11:117476] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n11:117476] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n06:117525] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n06:161721] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n06:161721] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n06:161721] [ 9] [b14n05:126781] *** Process received signal *** +[b14n05:126781] Signal: Segmentation fault (11) +[b14n05:126781] Signal code: (3) +[b14n05:126781] Failing at address: 0xb170000000000a0 +[b14n05:126781] [ 0] [0x1000000504d8] +[b14n05:126781] [ 1] [0x3fffdaaf0740] +[b14n05:126781] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n05:126781] [ 3] [a02n07:152084] *** Process received signal *** +[a02n07:152084] Signal: Segmentation fault (11) +[a02n07:152084] Signal code: (3) +[a02n07:152084] Failing at address: 0xb170000000000a0 +[a02n07:152084] [ 0] [0x1000000504d8] +[a02n07:152084] [ 1] [0x3ffff411c420] +[a02n07:152084] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n07:152084] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n12:178597] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n12:178597] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n12:178597] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n12:178597] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n13:10198] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n13:10198] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n13:10198] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n17:169738] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n17:169738] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n17:169738] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n17:169738] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n17:169738] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n17:169738] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n17:169738] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n05:139433] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n05:139433] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n05:139433] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n05:139433] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n05:139433] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n05:139433] [ 9] [0x1000000504d8] +[a31n04:158636] [ 1] [0x3fffc40e7fb0] +[a31n04:158636] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n04:158636] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n04:158636] [ 4] [b07n10:16972] *** Process received signal *** +[b07n10:16972] Signal: Segmentation fault (11) +[b07n10:16972] Signal code: (3) +[b07n10:16972] Failing at address: 0xb170000000000a0 +[b07n10:16972] [ 0] [0x1000000504d8] +[b07n10:16972] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n12:141260] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n12:141260] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n12:141260] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n12:141260] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n12:141260] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n13:153679] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n13:153679] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n13:153679] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n13:153679] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n13:153679] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n13:153679] *** End of error message *** +[b01n11:03984] *** Process received signal *** +[b01n11:03984] Signal: Segmentation fault (11) +[b01n11:03984] Signal code: (3) +[b01n11:03984] Failing at address: 0x14170000000000a0 +[b01n11:03984] [ 0] [0x1000000504d8] +[b01n11:03984] [ 1] [0x3ffff8db6240] +[b01n11:03984] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n07:168963] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n07:168963] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n07:168963] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n07:168963] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n07:168963] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n11:120580] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n11:120580] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n11:120580] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n04:145750] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n04:145750] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n17:177403] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n17:177403] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n17:177403] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n17:177403] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n17:177403] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n17:177403] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n17:177403] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n17:177403] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n17:177403] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n17:177403] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n17:177403] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n17:177403] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n16:08385] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n16:08385] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n16:08385] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n16:08385] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n10:11488] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n10:11488] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n10:11488] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n10:11488] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n13:171110] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n13:171110] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n13:171110] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n13:171110] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n14:178327] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n14:178327] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n14:178327] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n14:178327] [ 9] [b22n03:120668] *** Process received signal *** +[b22n03:120668] Signal: Segmentation fault (11) +[b22n03:120668] Signal code: (3) +[b22n03:120668] Failing at address: 0xb170000000000a0 +[b22n03:120668] [ 0] [0x1000000504d8] +[b22n03:120668] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n14:08398] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n14:08398] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n14:08398] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n16:148189] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n16:148189] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n16:148189] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n03:46687] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n03:46687] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n03:46687] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n03:46687] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n03:46687] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n03:46687] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n17:176828] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n17:176828] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n17:176828] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n17:176828] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n17:176828] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n17:176828] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n17:176828] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n17:176828] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n17:176828] *** End of error message *** +[b22n05:123934] *** Process received signal *** +[b22n05:123934] Signal: Segmentation fault (11) +[b22n05:123934] Signal code: (3) +[b22n05:123934] Failing at address: 0xb170000000000a0 +[b22n05:123934] [ 0] [0x1000000504d8] +[b22n05:123934] [ 1] [0x3fffe3ac79d0] +[b22n05:123934] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n18:179168] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n18:179168] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n18:179168] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n18:179168] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n13:154235] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n13:154235] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n13:154235] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n13:154235] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n13:154235] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n13:154235] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n07:160803] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n07:160803] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n07:160803] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n07:160803] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n07:160803] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n07:160803] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n07:160803] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n07:160803] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n07:160803] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n05:139910] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n05:139910] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n11:134522] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n11:134522] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n11:134522] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n11:134522] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n11:134522] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n04:44755] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n04:44755] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n04:44755] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n04:44755] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n12:123559] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n12:123559] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n12:123559] [ 9] [ 1] [0x3ffff6e69980] +[b03n03:06824] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n03:06824] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n03:06824] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n03:06824] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n03:06824] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n03:06824] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n03:06824] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n03:06824] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n09:119635] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n09:119635] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n09:119635] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n09:119635] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n05:156009] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n05:156009] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n13:123516] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n13:123516] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n13:123516] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n13:123516] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n13:123516] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n13:123516] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n13:123054] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n13:123054] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n13:123054] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n13:123054] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n13:123054] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n17:119978] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n17:119978] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n17:119978] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n17:119978] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n05:174781] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n05:174781] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n05:174781] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n10:169211] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n10:169211] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n10:169211] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n10:169211] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n10:169211] [ 9] [a30n07:165008] *** Process received signal *** +[a30n07:165008] Signal: Segmentation fault (11) +[a30n07:165008] Signal code: (3) +[a30n07:165008] Failing at address: 0xb170000000000a0 +[a30n07:165008] [ 0] [0x1000000504d8] +[a30n07:165008] [ 1] [0x3fffe138cf10] +[a30n07:165008] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n07:165008] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n07:165008] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n07:165008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n07:165008] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n07:165008] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n07:165008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n07:165008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n07:165008] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n07:165008] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n07:165008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n07:165008] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n07:165008] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n07:165008] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n04:03149] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n04:03149] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n04:03149] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n04:03149] [b09n08:176679] [ 0] [0x1000000504d8] +[b09n08:176679] [ 1] [0x3fffed54e7e0] +[b09n08:176679] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n08:176679] [ 3] [0x3ffff17a10e0] +[a26n16:53796] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n16:53796] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n16:53796] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n16:53796] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n16:53796] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n16:53796] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n16:53796] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n16:53796] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n16:53796] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n09:12247] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n09:12247] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n09:12247] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n14:133965] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n14:133965] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n14:133965] [ 5] [a34n03:151706] *** Process received signal *** +[a34n03:151706] Signal: Segmentation fault (11) +[a34n03:151706] Signal code: (3) +[a34n03:151706] Failing at address: 0x14170000000000a0 +[a34n03:151706] [ 0] [0x1000000504d8] +[a34n03:151706] [ 1] [0x3ffff89dfd60] +[a34n03:151706] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n03:151706] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n03:151706] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n03:151706] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n03:151706] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n03:151706] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n03:151706] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n03:151706] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n03:151706] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n03:151706] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n03:151706] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n03:151706] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n03:151706] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n03:151706] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n15:166120] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n15:166120] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n15:166120] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n15:166120] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n15:166120] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n15:166120] [10] [a30n01:163390] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n10:118387] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n10:118387] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n10:118387] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n10:118387] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n10:118387] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n12:160964] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n12:160964] [ 6] [a31n16:162625] *** Process received signal *** +[a31n16:162625] Signal: Segmentation fault (11) +[a31n16:162625] Signal code: (3) +[a31n16:162625] Failing at address: 0xb170000000000a0 +[a31n16:162625] [ 0] [0x1000000504d8] +[a31n16:162625] [ 1] [0x3fffd66b8710] +[a31n16:162625] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n16:162625] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n16:162625] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n16:162625] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n16:162625] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n16:162625] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n16:162625] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n16:162625] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n16:162625] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n16:162625] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n16:162625] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n16:162625] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n16:162625] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n16:162625] *** End of error message *** +[a28n06:179455] *** Process received signal *** +[a28n06:179455] Signal: Segmentation fault (11) +[a28n06:179455] Signal code: (3) +[a28n06:179455] Failing at address: 0xb170000000000a0 +[a28n06:179455] [ 0] [0x1000000504d8] +[a28n06:179455] [ 1] [0x3fffe7405bf0] +[a28n06:179455] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n06:179455] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n06:179455] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n06:179455] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n06:179455] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n06:179455] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n06:179455] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n06:179455] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n06:179455] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n06:179455] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n06:179455] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n06:179455] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n06:179455] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n06:179455] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n01:96751] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n01:96751] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n01:170765] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n01:170765] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n01:170765] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n08:141873] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n08:141873] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n11:130807] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n11:130807] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n11:130807] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n11:130807] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n11:130807] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n11:130807] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n11:130807] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n11:130807] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n11:130807] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n11:130807] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n18:128540] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n18:128540] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n18:128540] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n18:128540] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n18:128540] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n18:123413] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n18:123413] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n18:123413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n18:123413] [13] [b23n08:116906] *** Process received signal *** +[b23n08:116906] Signal: Segmentation fault (11) +[b23n08:116906] Signal code: (3) +[b23n08:116906] Failing at address: 0x14170000000000a0 +[b23n08:116906] [ 0] [0x1000000504d8] +[b23n08:116906] [ 1] [0x3ffff3e493d0] +[b23n08:116906] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n08:116906] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n08:116906] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n08:116906] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n08:116906] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n08:116906] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n08:116906] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n08:116906] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n08:116906] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n08:116906] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n08:116906] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n08:116906] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n08:116906] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n08:116906] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n08:159021] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n08:159021] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n08:159021] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n08:159021] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n08:159021] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n12:145500] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n12:145500] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n12:145500] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n12:145500] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n12:145500] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n12:145500] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n12:145500] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n12:145500] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n07:140135] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n10:131274] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n10:131274] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n15:122082] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n15:122082] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n15:122082] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n16:154185] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n16:154185] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n16:154185] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n16:154185] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n05:04520] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n05:04520] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n05:04520] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n05:04520] [10] [0x3ffff52b8ad0] +[b14n08:127446] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n08:127446] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n08:127446] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n08:127446] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n15:120133] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n15:120133] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n15:120133] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n15:120133] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n03:172648] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n03:172648] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n17:138615] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n17:138615] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n05:120231] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n05:120231] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n05:120231] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n05:120231] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n05:120231] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n13:133623] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n13:133623] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n13:133623] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n13:133623] [ 8] [b06n09:168249] *** Process received signal *** +[b06n09:168249] Signal: Segmentation fault (11) +[b06n09:168249] Signal code: (3) +[b06n09:168249] Failing at address: 0x14170000000000a0 +[b06n09:168249] [ 0] [0x1000000504d8] +[b06n09:168249] [ 1] [0x3fffd77cc8a0] +[b06n09:168249] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n09:168249] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n09:168249] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n09:168249] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n09:168249] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n09:168249] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n09:168249] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n09:168249] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n09:168249] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n09:168249] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n09:168249] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n09:168249] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n09:168249] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n09:168249] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n18:149116] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n18:149116] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n18:149116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n18:149116] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n18:149116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n18:149116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n18:149116] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n18:149116] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n18:149116] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n18:149116] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n18:149116] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n18:149116] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n10:126613] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n10:126613] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n10:126613] [13] [b17n06:119114] *** Process received signal *** +[b17n06:119114] Signal: Segmentation fault (11) +[b17n06:119114] Signal code: (3) +[b17n06:119114] Failing at address: 0x14170000000000a0 +[b17n06:119114] [ 0] [0x1000000504d8] +[b17n06:119114] [ 1] [0x3fffe4a71110] +[b17n06:119114] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n06:119114] [ 3] [a33n10:140412] *** End of error message *** +[b17n01:124277] [ 0] [0x1000000504d8] +[b17n01:124277] [ 1] [0x3fffcf6e2d90] +[b17n01:124277] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n01:124277] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n01:124277] [ 4] [0x3fffe08ca9e0] +[a30n18:10720] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n18:10720] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n18:10720] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n18:10720] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n18:10720] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n18:10720] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n18:10720] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n18:10720] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n18:10720] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n18:10720] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n18:10720] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n18:10720] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n18:10720] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n18:10720] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n16:166499] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n16:166499] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n16:166499] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n16:166499] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n16:166499] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n16:166499] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n16:166499] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n16:166499] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n16:166499] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n16:166499] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n16:166499] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n16:166499] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n08:176176] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n08:176176] *** End of error message *** +[a33n11:156155] *** Process received signal *** +[a33n11:156155] Signal: Segmentation fault (11) +[a33n11:156155] Signal code: (3) +[a33n11:156155] Failing at address: 0x14170000000000a0 +[a33n11:156155] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n02:122178] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n02:122178] [10] [b13n15:139747] *** Process received signal *** +[b13n15:139747] Signal: Segmentation fault (11) +[b13n15:139747] Signal code: (3) +[b13n15:139747] Failing at address: 0xb170000000000a0 +[b13n15:139747] [ 0] [0x1000000504d8] +[b13n15:139747] [ 1] [0x3ffffd4fcfe0] +[b13n15:139747] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n15:139747] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n10:01249] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n10:01249] *** End of error message *** +[b13n14:126465] *** Process received signal *** +[b13n14:126465] Signal: Segmentation fault (11) +[b13n14:126465] Signal code: (3) +[b13n14:126465] Failing at address: 0xb170000000000a0 +[b13n14:126465] [ 0] [0x1000000504d8] +[b13n14:126465] [ 1] [0x3fffe65ca620] +[b13n14:126465] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n14:126465] [ 3] [b13n06:133801] *** Process received signal *** +[b13n06:133801] Signal: Segmentation fault (11) +[b13n06:133801] Signal code: (3) +[b13n06:133801] Failing at address: 0xb170000000000a0 +[b13n06:133801] [ 0] [0x1000000504d8] +[b13n06:133801] [ 1] [0x3fffe1af1f10] +[b13n06:133801] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n06:133801] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n06:133801] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n06:133801] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n06:133801] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n06:133801] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n06:133801] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n06:133801] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n06:133801] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n06:133801] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n06:133801] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n06:133801] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n06:133801] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n06:133801] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n14:51450] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n14:51450] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n14:51450] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n14:51450] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n14:51450] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n14:51450] *** End of error message *** +[b10n15:164415] *** Process received signal *** +[b10n15:164415] Signal: Segmentation fault (11) +[b10n15:164415] Signal code: (3) +[b10n15:164415] Failing at address: 0xb170000000000a0 +[b10n15:164415] [ 0] [0x1000000504d8] +[b10n15:164415] [ 1] [0x3ffff2479ee0] +[b10n15:164415] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n15:164415] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n15:164415] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n15:164415] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n15:164415] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n15:164415] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n15:164415] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n15:164415] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n15:164415] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n15:164415] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n15:164415] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n15:164415] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n15:164415] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n15:164415] *** End of error message *** +[a34n07:165090] *** Process received signal *** +[a34n07:165090] Signal: Segmentation fault (11) +[a34n07:165090] Signal code: (3) +[a34n07:165090] Failing at address: 0xb170000000000a0 +[a34n07:165090] [ 0] [0x1000000504d8] +[a34n07:165090] [ 1] [0x3fffdcad2860] +[a34n07:165090] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n07:165090] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n09:160374] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n09:160374] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n03:09642] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n03:09642] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n03:09642] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n03:09642] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n03:09642] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n10:10923] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n10:10923] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n10:10923] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n02:74432] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n02:74432] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n02:74432] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n02:74432] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n02:74432] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n02:74432] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n02:74432] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n02:74432] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n02:74432] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n02:74432] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n10:120976] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n10:120976] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n10:120976] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n10:120976] [13] [0x3ffffc613dd0] +[a31n10:155397] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n10:155397] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n10:155397] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n10:155397] [ 5] [0x3fffc8267180] +[b09n10:169684] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n10:169684] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n10:169684] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n10:169684] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n10:169684] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n10:169684] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n10:169684] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n10:169684] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n10:169684] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n10:169684] [11] [a32n07:172876] *** Process received signal *** +[a32n07:172876] Signal: Segmentation fault (11) +[a32n07:172876] Signal code: (3) +[a32n07:172876] Failing at address: 0xb170000000000a0 +[a32n07:172876] [ 0] [0x1000000504d8] +[a32n07:172876] [ 1] [0x3fffc6faaf10] +[a32n07:172876] [ 2] [a32n14:152747] *** Process received signal *** +[b19n01:121743] *** Process received signal *** +[b19n01:121743] Signal: Segmentation fault (11) +[b19n01:121743] Signal code: (3) +[b19n01:121743] Failing at address: 0xb170000000000a0 +[b19n01:121743] [ 0] [0x1000000504d8] +[b19n01:121743] [ 1] [0x3fffeff98830] +[b19n01:121743] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n01:121743] [ 3] [b09n13:164690] *** Process received signal *** +[b09n13:164690] Signal: Segmentation fault (11) +[b09n13:164690] Signal code: (3) +[b09n13:164690] Failing at address: 0xb170000000000a0 +[b09n13:164690] [ 0] [0x1000000504d8] +[b09n13:164690] [ 1] [0x3fffc7ae0f10] +[b09n13:164690] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n13:164690] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n13:164690] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n13:164690] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n13:164690] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n13:164690] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n13:164690] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n13:164690] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n13:164690] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n13:164690] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n13:164690] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n13:164690] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n13:164690] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n13:164690] *** End of error message *** +[b05n11:176580] *** Process received signal *** +[b05n11:176580] Signal: Segmentation fault (11) +[b05n11:176580] Signal code: (3) +[b05n11:176580] Failing at address: 0xb170000000000a0 +[b05n11:176580] [ 0] [0x1000000504d8] +[b05n11:176580] [ 1] [0x3fffdc7f9d30] +[b05n11:176580] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n11:176580] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n11:176580] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n11:176580] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n11:176580] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n11:176580] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n11:176580] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n11:176580] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n11:176580] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n11:176580] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n11:176580] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n11:176580] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n11:176580] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n11:176580] *** End of error message *** +[a31n18:169361] *** Process received signal *** +[a31n18:169361] Signal: Segmentation fault (11) +[a31n18:169361] Signal code: (3) +[a31n18:169361] Failing at address: 0x14170000000000a0 +[a31n18:169361] [ 0] [0x1000000504d8] +[a31n18:169361] [ 1] [0x3fffc0ef3040] +[a31n18:169361] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n18:169361] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n18:169361] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n18:169361] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n18:169361] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n18:169361] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n18:169361] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n18:169361] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n18:169361] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n18:169361] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n18:169361] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n18:169361] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n18:169361] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n18:169361] *** End of error message *** +[ 1] [0x3fffe63b6110] +[b13n02:125267] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n02:125267] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n02:125267] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n02:125267] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n02:125267] [ 6] [a31n14:165663] *** Process received signal *** +[a31n14:165663] Signal: Segmentation fault (11) +[a31n14:165663] Signal code: (3) +[a31n14:165663] Failing at address: 0x14170000000000a0 +[a31n14:165663] [ 0] [0x1000000504d8] +[a31n14:165663] [ 1] [0x3ffffc8baa90] +[a31n14:165663] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n14:165663] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n14:165663] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n14:165663] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n14:165663] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n14:165663] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n14:165663] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n14:165663] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n14:165663] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n14:165663] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n14:165663] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n14:165663] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n14:165663] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n14:165663] *** End of error message *** +[b05n04:06837] *** Process received signal *** +[b05n04:06837] Signal: Segmentation fault (11) +[b05n04:06837] Signal code: (3) +[b05n04:06837] Failing at address: 0x5170000000000a0 +[b05n04:06837] [ 0] [0x1000000504d8] +[b05n04:06837] [ 1] [0x3fffcdb5fc00] +[b05n04:06837] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n04:06837] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n04:06837] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n04:06837] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n04:06837] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n04:06837] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n04:06837] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n04:06837] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n04:06837] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n04:06837] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n04:06837] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n04:06837] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n04:06837] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n04:06837] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n05:173318] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n05:173318] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n05:173318] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n05:173318] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n05:173318] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n05:173318] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n05:173318] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n10:135852] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n10:135852] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n17:122356] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n17:122356] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n17:122356] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n17:122356] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n17:122356] [ 9] [a28n07:04682] *** Process received signal *** +[a28n07:04682] Signal: Segmentation fault (11) +[a28n07:04682] Signal code: (3) +[a28n07:04682] Failing at address: 0x14170000000000a0 +[a28n07:04682] [ 0] [0x1000000504d8] +[a28n07:04682] [ 1] [0x3fffd8aaacf0] +[a28n07:04682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n07:04682] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n07:04682] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n07:04682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n07:04682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n07:04682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n07:04682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n07:04682] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n07:04682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n07:04682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n07:04682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n07:04682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n07:04682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n07:04682] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n12:122872] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n12:122872] [10] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n02:158419] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n02:158419] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n02:158419] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n02:158419] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n02:158419] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n02:158419] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n02:158419] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n02:158419] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n02:158419] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n02:158419] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n02:158419] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n02:158419] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n02:158419] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n03:164496] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n03:164496] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n03:164496] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n03:164496] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n03:164496] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n07:45305] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n07:45305] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n07:45305] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n04:130749] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n04:130749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n04:130749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n04:130749] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n16:124959] *** End of error message *** +[a31n13:155850] *** Process received signal *** +[a31n13:155850] Signal: Segmentation fault (11) +[a31n13:155850] Signal code: (3) +[a31n13:155850] Failing at address: 0x14170000000000a0 +[a31n13:155850] [ 0] [0x1000000504d8] +[a31n13:155850] [ 1] [0x3fffeae43300] +[a31n13:155850] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n17:163178] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n17:163178] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n17:163178] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n17:163178] [12] [0x3fffc92abc50] +[b13n04:129407] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n04:129407] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n04:129407] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n04:129407] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n04:129407] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n04:129407] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n04:129407] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n04:129407] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n04:129407] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n04:129407] [11] [a35n05:149465] Signal: Segmentation fault (11) +[a35n05:149465] Signal code: (3) +[a35n05:149465] Failing at address: 0x14170000000000a0 +[a35n05:149465] [ 0] [0x1000000504d8] +[a35n05:149465] [ 1] [0x3fffc114ed80] +[a35n05:149465] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n05:149465] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n14:177511] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n14:177511] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n14:177511] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n14:177511] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n14:177511] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n14:177511] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n14:177511] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n14:177511] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n14:177511] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n14:177511] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n14:177511] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n14:177511] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n14:177511] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n05:176910] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n05:176910] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n05:176910] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n09:106789] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n09:106789] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n09:106789] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n09:170657] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n09:170657] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n09:170657] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n09:170657] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n09:170657] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n07:120697] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n07:120697] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n07:120697] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n07:120697] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n07:120697] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n03:135036] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n03:135036] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n03:135036] [ 9] [b15n10:122757] *** Process received signal *** +[b15n10:122757] Signal: Segmentation fault (11) +[b15n10:122757] Signal code: (3) +[b15n10:122757] Failing at address: 0xb170000000000a0 +[b15n10:122757] [ 0] [0x1000000504d8] +[b15n10:122757] [ 1] [0x3fffd37d87a0] +[b15n10:122757] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n10:122757] [ 3] [0x3fffe0197120] +[b07n02:174057] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n02:174057] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n02:174057] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n02:174057] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n11:123521] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n11:123521] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n11:123521] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n11:123521] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n09:128494] *** End of error message *** +[b17n14:126118] *** Process received signal *** +[b17n14:126118] Signal: Segmentation fault (11) +[b17n14:126118] Signal code: (3) +[b17n14:126118] Failing at address: 0x14170000000000a0 +[b17n14:126118] [ 0] [0x1000000504d8] +[b17n14:126118] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n08:118553] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n08:118553] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n08:118553] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n08:118553] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n08:118553] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n08:118553] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n07:02841] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n07:02841] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n07:02841] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n12:160189] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n12:160189] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n12:160189] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n12:160189] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n12:160189] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n02:139631] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n02:139631] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n02:139631] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n02:139631] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n02:139631] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n02:139631] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n02:125370] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n02:125370] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n02:125370] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n02:125370] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n08:146119] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n08:146119] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n08:146119] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n08:146119] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n04:136995] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n04:136995] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n04:136995] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n04:136995] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n04:136995] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n04:136995] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n04:136995] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n04:136995] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n04:136995] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n04:136995] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n04:136995] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n04:136995] *** End of error message *** +[0x3fffddcfeb50] +[a29n04:156413] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n04:156413] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n04:156413] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n03:119829] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n03:119829] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n03:119829] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n03:126186] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n03:126186] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n15:165474] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n15:165474] [ 5] [a26n03:68831] *** Process received signal *** +[a26n03:68831] Signal: Segmentation fault (11) +[a26n03:68831] Signal code: (3) +[a26n03:68831] Failing at address: 0x14170000000000a0 +[a26n03:68831] [ 0] [0x1000000504d8] +[a26n03:68831] [ 1] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n03:147537] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n03:147537] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n03:147537] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n03:147537] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n03:147537] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n03:147537] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n13:137744] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n13:137744] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n13:137744] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n13:137744] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n13:137744] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n13:137744] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n13:137744] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n13:137744] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n13:137744] [11] [0x3ffff1a8cfb0] +[b06n10:176261] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n10:176261] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n10:176261] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n10:176261] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n10:176261] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n10:176261] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n10:176261] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n10:176261] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n10:176261] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n10:176261] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n10:176261] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n10:176261] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n11:134748] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n11:134748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n11:134748] [ 6] [a29n13:165386] *** Process received signal *** +[a29n13:165386] Signal: Segmentation fault (11) +[a29n13:165386] Signal code: (3) +[a29n13:165386] Failing at address: 0xb170000000000a0 +[a29n13:165386] [ 0] [0x1000000504d8] +[a29n13:165386] [ 1] [0x3fffda9e98a0] +[a29n13:165386] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n13:165386] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n08:165282] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n08:165282] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n18:123866] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n18:123866] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n18:123866] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n18:123866] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n18:123866] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n18:123866] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n07:156486] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n07:156486] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n07:156486] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n07:156486] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n08:124116] *** End of error message *** +[b18n11:128954] *** Process received signal *** +[b18n11:128954] Signal: Segmentation fault (11) +[b18n11:128954] Signal code: (3) +[b18n11:128954] Failing at address: 0x14170000000000a0 +[b18n11:128954] [ 0] [0x1000000504d8] +[b18n11:128954] [ 1] [0x3fffde95d5f0] +[b18n11:128954] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n11:128954] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n01:123574] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n01:123574] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n01:123574] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n01:123574] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n01:123574] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n01:123574] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n14:122444] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n14:122444] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n14:122444] [ 5] [0x3fffcd4d51d0] +[b09n05:173823] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n05:173823] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n05:173823] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n05:173823] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n05:173823] [ 6] [b18n07:125451] *** Process received signal *** +[b18n07:125451] Signal: Segmentation fault (11) +[b18n07:125451] Signal code: (3) +[b18n07:125451] Failing at address: 0x14170000000000a0 +[b18n07:125451] [ 0] [0x1000000504d8] +[b18n07:125451] [ 1] [0x3fffe62d35d0] +[b18n07:125451] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n07:125451] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n07:163253] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n07:163253] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n07:163253] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n16:129237] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n16:129237] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n16:129237] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n16:129237] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n16:129237] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n16:129237] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n16:129237] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n16:129237] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n16:129237] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n12:145262] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n12:145262] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n12:145262] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n12:145262] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n12:145262] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n12:145262] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n12:145262] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n12:145262] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n12:145262] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n12:145262] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n06:148310] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n06:148310] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n06:148310] [ 8] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n04:136610] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n04:136610] *** End of error message *** +[b20n03:124681] *** Process received signal *** +[b20n03:124681] Signal: Segmentation fault (11) +[b20n03:124681] Signal code: (3) +[b20n03:124681] Failing at address: 0x14170000000000a0 +[b20n03:124681] [ 0] [0x1000000504d8] +[b20n03:124681] [ 1] [0x3fffe0f53c90] +[b20n03:124681] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n03:124681] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n12:168645] *** End of error message *** +[b18n12:120405] *** Process received signal *** +[b18n12:120405] Signal: Segmentation fault (11) +[b18n12:120405] Signal code: (3) +[b18n12:120405] Failing at address: 0x14170000000000a0 +[b18n12:120405] [ 0] [0x1000000504d8] +[b18n12:120405] [ 1] [0x3ffff11ecf70] +[b18n12:120405] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n12:120405] [ 3] [0x3ffff8d25ad0] +[a29n03:155728] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n03:155728] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n03:155728] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n03:155728] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n03:155728] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n03:155728] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n03:155728] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n03:155728] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n03:155728] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n07:116115] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n07:116115] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n07:116115] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n07:116115] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n07:116115] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n07:116115] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n07:116115] *** End of error message *** +[b11n07:156139] *** Process received signal *** +[b11n07:156139] Signal: Segmentation fault (11) +[b11n07:156139] Signal code: (3) +[b11n07:156139] Failing at address: 0x14170000000000a0 +[b11n07:156139] [ 0] [0x1000000504d8] +[b11n07:156139] [ 1] [0x3ffff8310200] +[b11n07:156139] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n07:156139] [ 3] [0x3fffd0983840] +[b22n14:116411] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n14:116411] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n14:116411] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n14:116411] [ 5] [b08n04:172233] *** Process received signal *** +[b08n04:172233] Signal: Segmentation fault (11) +[b08n04:172233] Signal code: (3) +[b08n04:172233] Failing at address: 0x14170000000000a0 +[b08n04:172233] [ 0] [0x1000000504d8] +[b08n04:172233] [ 1] [0x3ffff3e955b0] +[b08n04:172233] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n04:172233] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n04:172233] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n04:172233] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n04:172233] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n04:172233] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n04:172233] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n04:172233] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n04:172233] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n04:172233] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n04:172233] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n04:172233] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n04:172233] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n04:172233] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n14:177224] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n14:177224] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n14:177224] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n14:177224] [12] [b01n02:01429] *** Process received signal *** +[b01n02:01429] Signal: Segmentation fault (11) +[b01n02:01429] Signal code: (3) +[b01n02:01429] Failing at address: 0x14170000000000a0 +[b01n02:01429] [ 0] [0x1000000504d8] +[b01n02:01429] [ 1] [0x3fffc413cac0] +[b01n02:01429] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n02:01429] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n02:01429] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n02:01429] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n02:01429] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n02:01429] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n02:01429] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n02:01429] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n02:01429] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n02:01429] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n02:01429] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n02:01429] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n02:01429] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n02:01429] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n18:121303] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n18:121303] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n18:121303] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n18:121303] [13] [a27n05:53372] *** Process received signal *** +[a27n05:53372] Signal: Segmentation fault (11) +[a27n05:53372] Signal code: (3) +[a27n05:53372] Failing at address: 0xb170000000000a0 +[a27n05:53372] [ 0] [0x1000000504d8] +[a27n05:53372] [ 1] [b03n05:05607] *** Process received signal *** +[b03n05:05607] Signal: Segmentation fault (11) +[b03n05:05607] Signal code: (3) +[b03n05:05607] Failing at address: 0x14170000000000a0 +[b03n05:05607] [b02n09:166574] *** Process received signal *** +[b02n09:166574] Signal: Segmentation fault (11) +[b02n09:166574] Signal code: (3) +[b02n09:166574] Failing at address: 0x14170000000000a0 +[b02n09:166574] [ 0] [0x1000000504d8] +[b02n09:166574] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n04:177599] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n04:177599] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n04:177599] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n04:177599] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n04:177599] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n12:177497] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n12:177497] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n12:177497] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n12:177497] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n12:177497] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n12:177497] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n12:177497] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n12:177497] [13] [a32n16:147286] *** Process received signal *** +[a32n16:147286] Signal: Segmentation fault (11) +[a32n16:147286] Signal code: (3) +[a32n16:147286] Failing at address: 0x14170000000000a0 +[a32n16:147286] [ 0] [0x1000000504d8] +[a32n16:147286] [ 1] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n04:168490] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n04:168490] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n04:168490] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n04:168490] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n04:168490] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n04:168490] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n04:168490] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n04:168490] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n04:168490] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n04:168490] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n04:117354] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n04:117354] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n04:117354] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n04:117354] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n04:117354] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n14:139845] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n14:139845] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n14:139845] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n13:54242] *** Process received signal *** +[a26n13:54242] Signal: Segmentation fault (11) +[a26n13:54242] Signal code: (3) +[a26n13:54242] Failing at address: 0x14170000000000a0 +[a26n13:54242] [ 0] [0x1000000504d8] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n01:122016] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n01:122016] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n01:122016] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n01:122016] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n01:122016] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n05:121826] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n05:121826] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n05:121826] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n05:121826] [ 8] [0x3fffe8be02f0] +[b01n13:177638] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n13:177638] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n13:177638] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n13:177638] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n16:168050] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n16:168050] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n16:168050] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n16:168050] [14] [b13n08:144854] *** Process received signal *** +[b13n08:144854] Signal: Segmentation fault (11) +[b13n08:144854] Signal code: (3) +[b13n08:144854] Failing at address: 0xb170000000000a0 +[b13n08:144854] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n07:149477] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n07:149477] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n07:149477] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n07:149477] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n07:149477] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n07:149477] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n07:149477] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n07:149477] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n07:149477] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n07:149477] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n12:119805] [ 6] [0x3fffc8338ee0] +[b02n18:03311] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n18:03311] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n18:03311] [ 4] [b13n09:143900] *** Process received signal *** +[b13n09:143900] Signal: Segmentation fault (11) +[b13n09:143900] Signal code: (3) +[b13n09:143900] Failing at address: 0xb170000000000a0 +[b13n09:143900] [ 0] [0x1000000504d8] +[b13n09:143900] [ 1] [0x3fffe58e4de0] +[b13n09:143900] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n09:143900] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n09:143900] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n09:143900] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n09:143900] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n09:143900] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n09:143900] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n09:143900] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n09:143900] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n09:143900] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n09:143900] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n09:143900] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n09:143900] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n09:143900] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n13:122188] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n13:122188] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n13:122188] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n13:122188] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n14:120638] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n14:120638] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n14:120638] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n14:120638] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n14:120638] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n14:120638] [10] [0x1000000504d8] +[a35n08:138175] [ 1] [0x3fffd5d2b220] +[a35n08:138175] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n08:138175] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n08:138175] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n08:138175] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n16:130375] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n16:130375] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n16:130375] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n16:130375] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n18:122556] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n18:122556] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n18:122556] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n18:122556] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n18:122556] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n17:125809] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n17:125809] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n17:125809] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n10:126194] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n10:126194] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n10:126194] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n07:174562] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n07:174562] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n07:174562] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n07:174562] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n07:174562] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n12:180116] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n12:180116] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n12:180116] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n12:180116] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n12:180116] *** End of error message *** +[b09n07:173043] *** Process received signal *** +[b09n07:173043] Signal: Segmentation fault (11) +[b09n07:173043] Signal code: (3) +[b09n07:173043] Failing at address: 0xb170000000000a0 +[b09n07:173043] [ 0] [0x1000000504d8] +[b09n07:173043] [ 1] [0x3fffc14b6660] +[b09n07:173043] [ 2] [0x1000000504d8] +[a35n13:138090] [ 1] [0x3ffff05d6480] +[a35n13:138090] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n13:138090] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n13:138090] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n07:08397] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n18:176808] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n18:176808] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n11:124445] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n11:124445] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n11:124445] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n11:124445] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n14:136137] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n14:136137] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n14:136137] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n15:120058] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n15:120058] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n15:120058] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n15:120058] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n15:120058] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n10:141234] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n10:141234] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n14:150134] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n14:150134] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n12:164381] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n12:164381] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n12:164381] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n01:163024] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n01:163024] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n01:163024] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n01:163024] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n01:163024] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n01:163024] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n01:163024] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n01:163024] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n01:163024] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n15:148166] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n15:148166] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n15:151251] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n15:151251] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n09:173061] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n09:173061] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n09:173061] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n09:173061] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n09:173061] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n09:173061] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n09:173061] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n09:173061] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n09:173061] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n09:173061] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n10:124922] *** Process received signal *** +[b14n10:124922] Signal: Segmentation fault (11) +[b14n10:124922] Signal code: (3) +[b14n10:124922] Failing at address: 0x14170000000000a0 +[b14n10:124922] [ 0] [0x1000000504d8] +[b14n10:124922] [ 1] [0x3fffc7872e10] +[b14n10:124922] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n10:124922] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n10:124922] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n10:124922] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n10:124922] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n10:124922] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n10:124922] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n10:124922] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n10:124922] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n10:124922] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n10:124922] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n10:124922] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n10:124922] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n10:124922] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n18:173978] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n18:173978] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n18:173978] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n18:173978] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n18:173978] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n15:119252] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n15:119252] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n15:119252] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n15:119252] [14] [b21n14:119488] [ 0] [0x1000000504d8] +[b21n14:119488] [ 1] [0x3fffeb7f5250] +[b21n14:119488] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n14:119488] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n05:125614] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n05:125614] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n05:125614] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n05:125614] [14] [b20n04:126345] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n05:173849] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n05:173849] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n13:117511] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n13:117511] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n13:166926] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n13:166926] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n13:166926] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n13:166926] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n13:166926] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n13:166926] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n13:166926] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n17:168160] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n17:168160] [ 4] [b23n10:126791] *** Process received signal *** +[b23n10:126791] Signal: Segmentation fault (11) +[b23n10:126791] Signal code: (3) +[b23n10:126791] Failing at address: 0x14170000000000a0 +[b23n10:126791] [ 0] [0x1000000504d8] +[b23n10:126791] [ 1] [0x3fffd96b5850] +[b23n10:126791] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n10:126791] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n06:164014] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n06:164014] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n06:135850] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n01:178523] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n01:178523] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n01:178523] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n01:178523] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n01:178523] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n01:178523] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n01:178523] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n01:178523] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n01:178523] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n11:162472] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n11:162472] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n11:162472] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n11:162472] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n11:162472] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n02:177547] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n02:177547] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n02:177547] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n02:177547] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n02:177547] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n18:164796] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n18:164796] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n18:164796] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n18:164796] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n11:163259] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n11:163259] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n11:163259] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n04:153966] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n04:153966] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n04:153966] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n04:153966] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n04:153966] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n04:153966] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n04:153966] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n04:153966] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n12:178718] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n12:178718] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n12:178718] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n16:149062] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n16:149062] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n16:149062] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n10:134732] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n10:134732] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n10:134732] [ 9] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n18:08428] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n18:08428] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n02:119534] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n02:119534] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n02:119534] [ 5] [0x1000000504d8] +[a31n03:158656] [ 1] [0x3fffe8ec0700] +[a31n03:158656] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n03:158656] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n03:158656] [ 4] [a31n11:153416] *** Process received signal *** +[a31n11:153416] Signal: Segmentation fault (11) +[a31n11:153416] Signal code: (3) +[a31n11:153416] Failing at address: 0x5170000000000a0 +[a31n11:153416] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n18:177223] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n18:177223] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n18:177223] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n18:177223] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n18:177223] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n10:173682] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n10:173682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n05:164943] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n05:164943] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n05:164943] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n04:179174] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n04:179174] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n04:179174] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n04:179174] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n16:173548] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n16:173548] *** End of error message *** +[b14n06:127276] *** Process received signal *** +[b14n06:127276] Signal: Segmentation fault (11) +[b14n06:127276] Signal code: (3) +[b14n06:127276] Failing at address: 0xb170000000000a0 +[b14n06:127276] [ 0] [0x1000000504d8] +[b14n06:127276] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n12:120130] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n12:120130] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n11:124434] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n11:124434] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n11:124434] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n11:124434] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n17:129933] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n17:129933] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n09:128000] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n09:128000] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n09:128000] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n09:128000] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n09:128000] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n09:137775] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n09:137775] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n09:137775] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n09:137775] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n09:137775] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n09:137775] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n09:137775] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n09:137775] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n15:124803] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n15:124803] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n15:124803] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n15:124803] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n15:124803] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n18:140384] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n18:140384] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n18:140384] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n18:140384] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n18:140384] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n18:140384] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n18:140384] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n13:123888] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n13:123888] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n13:123888] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n13:123888] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n13:168161] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n13:168161] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n13:168161] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n13:168161] [ 9] [0x3fffea2e2360] +[b21n09:120693] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n09:120693] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n09:120693] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n09:120693] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n09:120693] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n09:120693] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n09:120693] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n09:120693] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n09:120693] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n09:120693] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n09:120693] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n09:120693] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n09:120693] [14] [0x1000000504d8] +[b14n18:131584] [ 1] [0x3fffc2c6ce40] +[b14n18:131584] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n18:131584] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n18:131584] [ 4] [ 0] [0x1000000504d8] +[b08n17:171198] [ 1] [0x3fffe06ac430] +[b08n17:171198] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n17:171198] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n17:171198] [ 4] [0x3fffffdc4c60] +[a27n08:54626] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n08:54626] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n08:54626] [ 4] [0x3fffcc541dc0] +[a32n10:162610] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n10:162610] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n10:162610] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n10:162610] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n10:162610] [ 6] [b19n14:133065] *** Process received signal *** +[b19n14:133065] Signal: Segmentation fault (11) +[b19n14:133065] Signal code: (3) +[b19n14:133065] Failing at address: 0x14170000000000a0 +[b19n14:133065] [ 0] [0x1000000504d8] +[b19n14:133065] [ 1] [0x3fffca7c03b0] +[b19n14:133065] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n14:133065] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n14:133065] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n09:161231] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n09:161231] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n09:161231] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n08:116171] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n08:116171] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n08:116171] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n08:116171] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n08:116171] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n08:131378] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n08:131378] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n08:131378] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n08:131378] [14] [0x1000000504d8] +[a27n04:52706] [ 1] [0x3fffeadcc6f0] +[a27n04:52706] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n04:52706] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n04:52706] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n04:52706] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n04:52706] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n04:52706] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n04:52706] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n04:52706] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n16:165920] [ 5] [b13n16:128438] *** Process received signal *** +[b13n16:128438] Signal: Segmentation fault (11) +[b13n16:128438] Signal code: (3) +[b13n16:128438] Failing at address: 0x14170000000000a0 +[b13n16:128438] [ 0] [a33n02:167599] *** Process received signal *** +[a33n02:167599] Signal: Segmentation fault (11) +[a33n02:167599] Signal code: (3) +[a33n02:167599] Failing at address: 0x14170000000000a0 +[a33n02:167599] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n07:117359] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n07:117359] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n07:133372] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n07:133372] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n07:133372] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n04:166207] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n04:166207] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n04:166207] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n04:166207] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n04:166207] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n04:166207] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n04:166207] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n04:166207] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n04:166207] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n04:166207] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n04:166207] *** End of error message *** +[b05n02:176051] *** Process received signal *** +[b05n02:176051] Signal: Segmentation fault (11) +[b05n02:176051] Signal code: (3) +[b05n02:176051] Failing at address: 0x5170000000000a0 +[b05n02:176051] [ 0] [0x1000000504d8] +[b05n02:176051] [ 1] [0x3fffc997c700] +[b05n02:176051] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n02:176051] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n04:41670] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n04:41670] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n04:41670] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n04:41670] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n04:41670] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n04:41670] [12] [0x3fffce5ae340] +[b17n16:127352] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n16:127352] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n16:127352] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n16:127352] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n16:127352] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n17:124166] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n17:124166] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n17:124166] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n08:76223] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n08:76223] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n08:76223] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n08:76223] [ 9] [0x1000000504d8] +[a29n18:07422] [ 1] [0x3fffcf64a2c0] +[a29n18:07422] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n18:07422] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n18:07422] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n18:07422] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n01:128829] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n01:128829] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n12:132099] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n06:167368] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n06:167368] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n18:69155] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n18:69155] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n18:69155] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n18:69155] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n14:176069] [ 4] [b19n03:123614] *** Process received signal *** +[b19n03:123614] Signal: Segmentation fault (11) +[b19n03:123614] Signal code: (3) +[b19n03:123614] Failing at address: 0xb170000000000a0 +[b19n03:123614] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n11:23062] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n11:23062] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n11:23062] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n11:23062] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n11:23062] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n11:23062] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n11:23062] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n11:23062] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n11:23062] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n11:23062] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n08:172899] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n08:172899] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n08:172899] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n08:172899] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n08:172899] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n08:172899] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n14:168375] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n14:168375] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n09:149100] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n09:149100] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n09:149100] [ 6] [b07n11:170896] *** Process received signal *** +[b07n11:170896] Signal: Segmentation fault (11) +[b07n11:170896] Signal code: (3) +[b07n11:170896] Failing at address: 0xb170000000000a0 +[b07n11:170896] [ 0] [0x1000000504d8] +[b07n11:170896] [ 1] [0x3fffe4d1c750] +[b07n11:170896] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n11:170896] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n13:122033] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n13:122033] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n13:122033] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n13:122033] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n13:122033] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n01:166960] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n01:166960] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n01:166960] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n01:166960] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n01:166960] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n01:166960] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n01:166960] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n01:166960] [11] [b02n07:180192] *** Process received signal *** +[b02n07:180192] Signal: Segmentation fault (11) +[b02n07:180192] Signal code: (3) +[b02n07:180192] Failing at address: 0x14170000000000a0 +[b02n07:180192] [ 0] [0x1000000504d8] +[b02n07:180192] [ 1] [0x3fffd9ae4d50] +[b02n07:180192] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n07:180192] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n11:133973] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n11:133973] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n11:133973] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n11:133973] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n11:133973] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n11:133973] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n11:133973] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n11:133973] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n11:133973] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n11:133973] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n18:21771] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n18:21771] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n18:21771] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n18:21771] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n18:21771] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n18:21771] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n03:38504] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n03:38504] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n03:38504] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n03:38504] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n03:38504] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n03:38504] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n03:38504] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n03:38504] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n03:38504] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n03:38504] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n03:38504] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n09:51331] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n09:51331] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n09:51331] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n09:51331] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n09:51331] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n09:51331] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n16:135791] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n16:135791] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n16:135791] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n16:135791] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n16:135791] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n16:135791] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n16:135791] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n16:135791] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n16:135791] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n01:48806] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n01:48806] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n01:48806] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n01:48806] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n01:48806] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n01:48806] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n13:114397] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n13:114397] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n13:114397] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n13:114397] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n13:114397] *** End of error message *** +[b08n12:12764] *** Process received signal *** +[b08n12:12764] Signal: Segmentation fault (11) +[b08n12:12764] Signal code: (3) +[b08n12:12764] Failing at address: 0xb170000000000a0 +[b08n12:12764] [ 0] [0x1000000504d8] +[b08n12:12764] [ 1] [0x3fffc2808df0] +[b08n12:12764] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n12:12764] [ 3] [0x1000000504d8] +[b10n11:160347] [ 1] [0x3fffd8fc55a0] +[b10n11:160347] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n11:160347] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n11:160347] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n12:167678] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n15:177059] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n15:177059] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n15:177059] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n06:126374] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n06:126374] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n06:126374] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n17:179116] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n17:179116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n10:142930] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n10:142930] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n10:142930] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n17:149904] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n17:149904] [13] [a31n15:155433] *** Process received signal *** +[a31n15:155433] Signal: Segmentation fault (11) +[a31n15:155433] Signal code: (3) +[a31n15:155433] Failing at address: 0x14170000000000a0 +[a31n15:155433] [ 0] [0x1000000504d8] +[a31n15:155433] [ 1] [0x3ffff043ff10] +[a31n15:155433] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n15:155433] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n15:155433] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n16:116525] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n16:116525] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n16:116525] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n16:116525] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n16:116525] [13] [0x3ffff5f3d020] +[b13n05:135166] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n05:135166] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n05:135166] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n05:135166] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n05:135166] [ 6] [a31n05:158576] *** Process received signal *** +[a31n05:158576] Signal: Segmentation fault (11) +[a31n05:158576] Signal code: (3) +[a31n05:158576] Failing at address: 0x14170000000000a0 +[a31n05:158576] [ 0] [0x1000000504d8] +[a31n05:158576] [ 1] [0x3fffe474a090] +[a31n05:158576] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n05:158576] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n05:158576] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n05:158576] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n05:158576] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n05:158576] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n05:158576] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n05:158576] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n05:158576] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n05:158576] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n05:158576] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n05:158576] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n05:158576] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n05:158576] *** End of error message *** +[0x1000000504d8] +[b22n08:118581] [ 1] [0x3fffcd630c70] +[b22n08:118581] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n08:118581] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n08:118581] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n08:118581] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n16:157888] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n16:157888] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n16:157888] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n16:157888] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n16:157888] [14] [b19n09:123418] *** Process received signal *** +[b19n09:123418] Signal: Segmentation fault (11) +[b19n09:123418] Signal code: (3) +[b19n09:123418] Failing at address: 0x14170000000000a0 +[b19n09:123418] [ 0] [0x1000000504d8] +[b19n09:123418] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n07:118166] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n07:118166] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n07:118166] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n07:118166] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n07:118166] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n14:128870] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n14:128870] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n14:128870] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n02:06561] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n02:06561] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n02:06561] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n18:179700] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n18:179700] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n18:179700] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n18:179700] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n17:03845] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n17:03845] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n17:03845] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n17:03845] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n17:03845] *** End of error message *** +[0x3ffff9031340] +[b02n01:167297] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n01:167297] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n01:167297] [ 4] [b14n13:122140] *** Process received signal *** +[b14n13:122140] Signal: Segmentation fault (11) +[b14n13:122140] Signal code: (3) +[b14n13:122140] Failing at address: 0xb170000000000a0 +[b14n13:122140] [ 0] [0x1000000504d8] +[b14n13:122140] [ 1] [0x3fffd7b54bf0] +[b14n13:122140] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n13:122140] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n14:133632] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n14:133632] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n14:133632] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n13:176366] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n13:176366] [ 5] [a28n09:03926] *** Process received signal *** +[a28n09:03926] Signal: Segmentation fault (11) +[a28n09:03926] Signal code: (3) +[a28n09:03926] Failing at address: 0x14170000000000a0 +[a28n09:03926] [ 0] [0x1000000504d8] +[a28n09:03926] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n14:169522] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n14:169522] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n12:122963] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n12:122963] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n12:122963] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n12:122963] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n12:122963] [ 9] [a28n10:176309] *** Process received signal *** +[a28n10:176309] Signal: Segmentation fault (11) +[a28n10:176309] Signal code: (3) +[a28n10:176309] Failing at address: 0x14170000000000a0 +[a28n10:176309] [ 0] [a27n17:15874] Signal: Segmentation fault (11) +[a27n17:15874] Signal code: (3) +[a27n17:15874] Failing at address: 0x14170000000000a0 +[a27n17:15874] [ 0] [0x1000000504d8] +[a27n17:15874] [ 1] [0x3fffeeb08130] +[a27n17:15874] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n17:15874] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n17:15874] [ 4] [a33n01:141819] *** Process received signal *** +[a33n01:141819] Signal: Segmentation fault (11) +[a33n01:141819] Signal code: (3) +[a33n01:141819] Failing at address: 0x14170000000000a0 +[a33n01:141819] [ 0] [0x1000000504d8] +[a33n01:141819] [ 1] [0x3ffff0f01260] +[a33n01:141819] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n06:01838] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n06:01838] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n06:01838] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n06:01838] *** End of error message *** +[b12n17:138141] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n17:138141] [ 4] [a35n07:138202] Signal: Segmentation fault (11) +[a35n07:138202] Signal code: (3) +[a35n07:138202] Failing at address: 0xb170000000000a0 +[a35n07:138202] [ 0] [0x1000000504d8] +[a35n07:138202] [ 1] [0x3fffc4240d10] +[a35n07:138202] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n07:138202] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n03:177925] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n03:177925] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n03:177925] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n14:175714] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n14:175714] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n14:175714] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n14:175714] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n06:143012] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n06:143012] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n06:143012] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n06:143012] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n14:159239] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n14:159239] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n14:159239] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n14:159239] [14] [b05n08:08956] *** Process received signal *** +[b05n08:08956] Signal: Segmentation fault (11) +[b05n08:08956] Signal code: (3) +[b05n08:08956] Failing at address: 0x5170000000000a0 +[b05n08:08956] [ 0] [0x1000000504d8] +[b05n08:08956] [ 1] [0x3fffca451820] +[b05n08:08956] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n08:08956] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n16:124664] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n16:124664] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n16:124664] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n16:124664] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n07:131671] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n07:131671] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n07:131671] [ 9] [0x1000000504d8] +[a35n14:136661] [ 1] [0x3fffd4368d40] +[a35n14:136661] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n14:136661] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n14:136661] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n14:136661] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n14:136661] [ 6] [b02n06:43018] *** Process received signal *** +[b02n06:43018] Signal: Segmentation fault (11) +[b02n06:43018] Signal code: (3) +[b02n06:43018] Failing at address: 0xb170000000000a0 +[b02n06:43018] [ 0] [0x1000000504d8] +[b02n06:43018] [ 1] [0x3fffc714bd20] +[b02n06:43018] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n06:43018] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n17:156181] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n17:156181] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n17:156181] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n17:156181] [ 8] [b17n18:126904] [ 0] [0x1000000504d8] +[b17n18:126904] [ 1] [0x3fffe98dccc0] +[b17n18:126904] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n18:126904] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n18:126904] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n07:132089] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n07:132089] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n07:132089] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n07:132089] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n10:67857] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n10:67857] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n10:67857] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n10:67857] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n10:67857] [10] [b09n11:171964] *** Process received signal *** +[b09n11:171964] Signal: Segmentation fault (11) +[b09n11:171964] Signal code: (3) +[b09n11:171964] Failing at address: 0xb170000000000a0 +[b09n11:171964] [ 0] [0x1000000504d8] +[b09n11:171964] [ 1] [a33n04:169912] *** Process received signal *** +[a33n04:169912] Signal: Segmentation fault (11) +[a33n04:169912] Signal code: (3) +[a33n04:169912] Failing at address: 0x14170000000000a0 +[a33n04:169912] [ 0] [0x1000000504d8] +[a33n04:169912] [ 1] [0x3fffc39de230] +[a33n04:169912] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n04:169912] [ 3] [0x1000000504d8] +[b10n02:164488] [ 1] [0x3ffff7a07510] +[b10n02:164488] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n02:164488] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n02:164488] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n07:153177] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n07:153177] [11] [b02n02:166746] [ 0] [0x1000000504d8] +[b02n02:166746] [ 1] [0x3fffffcf4fd0] +[b02n02:166746] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n02:166746] [ 3] [b02n16:01573] [10] [0x3fffdfca1540] +[b15n13:123709] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n13:123709] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n13:123709] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n13:123709] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n13:123709] [ 6] [a29n02:163026] *** Process received signal *** +[a29n02:163026] Signal: Segmentation fault (11) +[a29n02:163026] Signal code: (3) +[a29n02:163026] Failing at address: 0xb170000000000a0 +[a29n02:163026] [ 0] [0x1000000504d8] +[a29n02:163026] [ 1] [0x3fffd149d330] +[a29n02:163026] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n02:163026] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n02:163026] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n02:163026] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n02:163026] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n02:163026] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n02:163026] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n02:163026] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n02:163026] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n02:163026] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n02:163026] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n02:163026] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n02:163026] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n02:163026] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n08:147823] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n08:147823] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n08:147823] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n01:128157] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n01:128157] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n01:128157] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n03:10292] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n03:10292] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n03:10292] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n03:10292] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n13:156921] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n13:156921] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n13:156921] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n13:156921] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n11:152987] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n11:152987] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n11:152987] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n11:152987] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n11:152987] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n11:152987] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n11:152987] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n11:152987] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n11:152987] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n11:152987] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n11:152987] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n03:139011] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n03:139011] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n03:139011] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n03:139011] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n03:139011] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n03:139011] [ 9] [b12n05:138195] *** Process received signal *** +[b12n05:138195] Signal: Segmentation fault (11) +[b12n05:138195] Signal code: (3) +[b12n05:138195] Failing at address: 0x5170000000000a0 +[b12n05:138195] [ 0] [0x1000000504d8] +[b12n05:138195] [ 1] [0x3fffc8740560] +[b12n05:138195] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n05:138195] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n05:138195] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n05:138195] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n05:138195] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n05:138195] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n05:138195] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n05:138195] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n05:138195] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n05:138195] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n05:138195] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n05:138195] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n05:138195] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n05:138195] *** End of error message *** +[a27n18:28217] *** Process received signal *** +[a27n18:28217] Signal: Segmentation fault (11) +[a27n18:28217] Signal code: (3) +[a27n18:28217] Failing at address: 0x14170000000000a0 +[a27n18:28217] [ 0] [0x1000000504d8] +[a27n18:28217] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n15:85939] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n15:85939] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n15:85939] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n15:85939] [14] [b11n17:164296] *** Process received signal *** +[b11n17:164296] Signal: Segmentation fault (11) +[b11n17:164296] Signal code: (3) +[b11n17:164296] Failing at address: 0xb170000000000a0 +[b11n17:164296] [ 0] [0x1000000504d8] +[b11n17:164296] [ 1] [0x3fffe560caa0] +[b11n17:164296] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n17:164296] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n17:164296] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n17:164296] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n17:164296] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n17:164296] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n17:164296] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n17:164296] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n09:121755] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n09:121755] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n09:121755] [ 8] [a36n17:150928] *** Process received signal *** +[a36n17:150928] Signal: Segmentation fault (11) +[a36n17:150928] Signal code: (3) +[a36n17:150928] Failing at address: 0x5170000000000a0 +[a36n17:150928] [ 0] [0x1000000504d8] +[a36n17:150928] [ 1] [0x3fffdc7c1bd0] +[a36n17:150928] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n17:150928] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n11:137872] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n11:137872] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n11:137872] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n11:137872] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n11:137872] [10] [a36n11:152841] *** Process received signal *** +[a36n11:152841] Signal: Segmentation fault (11) +[a36n11:152841] Signal code: (3) +[a36n11:152841] Failing at address: 0xb170000000000a0 +[a36n11:152841] [ 0] [0x1000000504d8] +[a36n11:152841] [ 1] [0x3ffff5e44160] +[a36n11:152841] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n11:152841] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n11:01170] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n11:01170] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n08:124725] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n08:124725] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n08:124725] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n08:124725] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n08:124725] *** End of error message *** +[0x1000000504d8] +[b18n09:123812] [ 1] [0x3fffcefc9d40] +[b18n09:123812] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n09:123812] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n09:123812] [ 4] [0x3ffff6a3f0e0] +[b03n13:01298] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n13:01298] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n13:01298] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n13:01298] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n13:01298] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n02:143968] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n02:143968] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n02:143968] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n02:143968] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n02:143968] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n02:143968] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n02:143968] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n02:143968] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n02:143968] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n02:143968] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n02:143968] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n02:143968] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n10:130582] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n10:130582] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n16:153290] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n06:122164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n06:122164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n06:122164] [10] [b11n14:161618] Signal: Segmentation fault (11) +[b11n14:161618] Signal code: (3) +[b11n14:161618] Failing at address: 0xb170000000000a0 +[b11n14:161618] [ 0] [0x1000000504d8] +[b11n14:161618] [ 1] [0x3fffe1c61670] +[b11n14:161618] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n14:161618] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n12:11141] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n17:119582] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n17:119582] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n17:119582] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n09:119548] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n09:119548] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n09:119548] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n09:119548] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n09:119548] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n09:119548] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n09:119548] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n09:119548] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n09:119548] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n01:130410] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n01:130410] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n01:130410] [12] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n09:173749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n09:173749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n09:173749] [ 5] [0x3fffc306c360] +[b12n04:138158] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n04:138158] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n04:138158] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n04:138158] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n15:177860] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n15:177860] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n15:177860] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n15:177860] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n15:177860] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n15:177860] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n15:177860] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n15:177860] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n16:122495] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n16:122495] [ 5] [b18n10:122209] *** Process received signal *** +[b18n10:122209] Signal: Segmentation fault (11) +[b18n10:122209] Signal code: (3) +[b18n10:122209] Failing at address: 0x14170000000000a0 +[b18n10:122209] [ 0] [0x1000000504d8] +[b18n10:122209] [ 1] [0x3fffd5d9d300] +[b18n10:122209] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n10:122209] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n12:123233] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n12:123233] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n12:123233] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n03:127630] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n03:127630] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n03:127630] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n03:127630] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n03:127630] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n03:127630] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n03:127630] [14] [a30n12:148382] *** Process received signal *** +[b08n13:158866] *** Process received signal *** +[b08n13:158866] Signal: Segmentation fault (11) +[b08n13:158866] Signal code: (3) +[b08n13:158866] Failing at address: 0x14170000000000a0 +[b08n13:158866] [ 0] [0x1000000504d8] +[b08n13:158866] [ 1] [0x3ffff6a4bf30] +[b08n13:158866] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n13:158866] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n13:158866] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n13:158866] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n13:158866] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n13:158866] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n13:158866] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n13:158866] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n13:158866] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n13:158866] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n13:158866] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n13:158866] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n13:158866] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n13:158866] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n08:174411] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n08:174411] *** End of error message *** +[0x3ffff245c320] +[b12n01:145734] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n01:145734] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n01:145734] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n04:175999] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n04:175999] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n04:175999] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n04:175999] [ 9] [a33n15:148579] *** Process received signal *** +[a33n15:148579] Signal: Segmentation fault (11) +[a33n15:148579] Signal code: (3) +[a33n15:148579] Failing at address: 0xb170000000000a0 +[a33n15:148579] [ 0] [0x1000000504d8] +[a33n15:148579] [ 1] [0x3fffc786a010] +[a33n15:148579] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n15:148579] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n16:171918] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n16:171918] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n16:171918] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n16:171918] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n16:171918] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n16:171918] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n01:173778] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n01:173778] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n01:173778] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n01:173778] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n01:173778] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n17:164560] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n17:164560] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n17:164560] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n17:164560] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n10:151892] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n10:151892] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n10:151892] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n10:151892] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n10:151892] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n10:151892] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n10:151892] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n10:151892] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n10:151892] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n18:163107] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n18:163107] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n18:163107] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n18:163107] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n12:52453] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n12:52453] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n12:52453] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n12:52453] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n12:156918] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n12:156918] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n12:156918] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n12:156918] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n12:156918] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n12:156918] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n12:156918] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n12:156918] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n03:168210] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n03:168210] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n03:168210] [13] [0x3fffcca94560] +[b06n02:163322] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n02:163322] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n02:163322] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n02:163322] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n02:163322] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n18:163572] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n18:163572] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n18:163572] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n18:163572] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n18:163572] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n18:163572] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n18:163572] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n18:163572] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n18:163572] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n18:163572] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n18:163572] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n18:163572] *** End of error message *** +[0x3fffc639a690] +[b06n14:173896] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n14:173896] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n14:173896] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n14:173896] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n08:129889] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n08:129889] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n08:129889] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n09:122069] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n09:122069] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n09:122069] [ 6] [ 1] [0x3fffcef68750] +[b07n04:164046] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n04:164046] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n04:164046] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n11:176299] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n11:176299] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n11:176299] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n11:176299] [14] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n09:152062] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n09:152062] [ 4] [b09n06:167092] *** Process received signal *** +[b09n06:167092] Signal: Segmentation fault (11) +[b09n06:167092] Signal code: (3) +[b09n06:167092] Failing at address: 0xb170000000000a0 +[b09n06:167092] [ 0] [0x1000000504d8] +[b09n06:167092] [ 1] [0x3fffd9afa280] +[b09n06:167092] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n06:167092] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n06:167092] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n06:167092] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n06:167092] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n06:167092] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n06:167092] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n06:167092] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n06:167092] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n06:167092] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n06:167092] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n06:167092] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n06:167092] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n06:167092] *** End of error message *** +[0x1000000504d8] +[b14n07:127355] [ 1] [0x3fffff9f3ee0] +[b14n07:127355] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n07:127355] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n07:127355] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n07:127355] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n06:155087] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n06:155087] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n06:155087] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n06:155087] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n06:155087] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n06:155087] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n06:155087] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n06:155087] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n06:155087] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n06:155087] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n06:155087] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n06:155087] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n07:125472] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n07:125472] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n07:125472] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n02:120190] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n02:120190] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n02:120190] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n02:120190] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n08:163161] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n08:163161] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n08:163161] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n08:163161] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n11:147712] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n11:147712] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n11:147712] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n11:147712] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n01:165166] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n01:165166] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n15:03733] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n15:03733] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n15:03733] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n15:03733] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n15:03733] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n15:03733] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n15:03733] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n15:03733] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n15:03733] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n15:03733] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n15:03733] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n15:03733] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n08:156372] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n08:156372] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n08:156372] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n04:169414] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n04:169414] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n04:169414] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n04:169414] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n04:169414] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n04:169414] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n13:165260] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n13:165260] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n13:165260] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n13:165260] *** End of error message *** +[a27n06:60150] *** Process received signal *** +[a27n06:60150] Signal: Segmentation fault (11) +[a27n06:60150] Signal code: (3) +[a27n06:60150] Failing at address: 0x14170000000000a0 +[a27n06:60150] [ 0] [0x1000000504d8] +[a27n06:60150] [ 1] [0x3ffff59b2210] +[a27n06:60150] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n06:60150] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n06:60150] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n06:60150] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n06:60150] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n06:60150] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n06:60150] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n06:60150] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n06:60150] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n06:60150] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n06:60150] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n06:60150] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n06:60150] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n06:60150] *** End of error message *** +[b03n11:175595] *** Process received signal *** +[b03n11:175595] Signal: Segmentation fault (11) +[b03n11:175595] Signal code: (3) +[b03n11:175595] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n16:166690] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n16:166690] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n16:166690] [ 6] [a30n02:155380] *** Process received signal *** +[a30n02:155380] Signal: Segmentation fault (11) +[a30n02:155380] Signal code: (3) +[a30n02:155380] Failing at address: 0xb170000000000a0 +[a30n02:155380] [ 0] [0x1000000504d8] +[a30n02:155380] [ 1] [0x3fffcbd8a640] +[a30n02:155380] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n02:155380] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n02:155380] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n08:163101] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n08:163101] *** End of error message *** +[0x3ffff7283660] +[a35n18:145742] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n18:145742] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n18:145742] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n18:145742] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n18:145742] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n18:145742] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n18:145742] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n18:145742] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n18:145742] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n15:154385] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n15:154385] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n15:154385] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n15:154385] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n15:154385] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n15:154385] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n15:154385] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n15:154385] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n15:154385] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n15:154385] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n15:154385] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n12:115637] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n12:115637] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n12:115637] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n12:115637] [ 9] [a34n05:148378] *** Process received signal *** +[a34n05:148378] Signal: Segmentation fault (11) +[a34n05:148378] Signal code: (3) +[a34n05:148378] Failing at address: 0x14170000000000a0 +[a34n05:148378] [ 0] [0x1000000504d8] +[a34n05:148378] [ 1] [0x3fffc02b4e00] +[a34n05:148378] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n05:148378] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n05:148378] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n06:38119] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n06:38119] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n06:38119] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n06:38119] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n06:38119] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n06:38119] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n06:38119] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n06:38119] [10] [a32n13:163194] *** Process received signal *** +[a32n13:163194] Signal: Segmentation fault (11) +[a32n13:163194] Signal code: (3) +[a32n13:163194] Failing at address: 0x14170000000000a0 +[a32n13:163194] [ 0] [0x1000000504d8] +[a32n13:163194] [ 1] [0x3fffe6009180] +[a32n13:163194] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n13:163194] [ 3] [0x3fffdee67270] +[a35n01:138003] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n01:138003] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n01:138003] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n01:138003] [b05n06:177847] *** Process received signal *** +[b05n06:177847] Signal: Segmentation fault (11) +[b05n06:177847] Signal code: (3) +[b05n06:177847] Failing at address: 0xb170000000000a0 +[b05n06:177847] [ 0] [0x1000000504d8] +[b05n06:177847] [ 1] [0x3fffe5b4adf0] +[b05n06:177847] [b04n05:169786] *** Process received signal *** +[b04n05:169786] Signal: Segmentation fault (11) +[b04n05:169786] Signal code: (3) +[b04n05:169786] Failing at address: 0xb170000000000a0 +[b04n05:169786] [ 0] [0x1000000504d8] +[b04n05:169786] [ 1] [0x3fffcddcce10] +[b04n05:169786] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n05:169786] [ 3] [b07n15:166704] *** Process received signal *** +[b07n15:166704] Signal: Segmentation fault (11) +[b07n15:166704] Signal code: (3) +[b07n15:166704] Failing at address: 0xb170000000000a0 +[b07n15:166704] [ 0] [0x1000000504d8] +[b07n15:166704] [ 1] [0x3fffeb0a4d30] +[b07n15:166704] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n15:166704] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n15:166704] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n15:166704] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n15:166704] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n15:166704] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n15:166704] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n15:166704] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n15:166704] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n15:166704] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n15:166704] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n15:166704] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n15:166704] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n15:166704] *** End of error message *** +[0x3ffff7aa5f80] +[a29n09:79007] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n09:79007] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n09:79007] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n09:79007] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n12:154341] [ 6] [b02n03:39685] *** Process received signal *** +[b02n03:39685] Signal: Segmentation fault (11) +[b02n03:39685] Signal code: (3) +[b02n03:39685] Failing at address: 0xb170000000000a0 +[b02n03:39685] [ 0] [0x1000000504d8] +[b02n03:39685] [ 1] [0x3fffe5a536e0] +[b02n03:39685] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n03:39685] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n03:39685] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n03:39685] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n03:39685] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n03:39685] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n03:39685] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n03:39685] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n03:39685] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n03:39685] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n03:39685] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n03:39685] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n03:39685] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n03:39685] *** End of error message *** +[b19n08:121690] *** Process received signal *** +[b19n08:121690] Signal: Segmentation fault (11) +[b19n08:121690] Signal code: (3) +[b19n08:121690] Failing at address: 0x14170000000000a0 +[b19n08:121690] [ 0] [0x1000000504d8] +[b19n08:121690] [ 1] [0x3fffeb648200] +[b19n08:121690] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n08:121690] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n11:120242] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n11:120242] [ 6] [0x3fffdf89d2b0] +[a35n02:149226] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n02:149226] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n02:149226] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n02:149226] [ 5] [b04n06:10226] *** Process received signal *** +[b04n06:10226] Signal: Segmentation fault (11) +[b04n06:10226] Signal code: (3) +[b04n06:10226] Failing at address: 0x5170000000000a0 +[b04n06:10226] [ 0] [0x1000000504d8] +[b04n06:10226] [ 1] [0x3fffe4db36a0] +[b04n06:10226] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n15:137007] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n15:137007] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n15:137007] [ 8] [b06n12:177119] *** Process received signal *** +[b06n12:177119] Signal: Segmentation fault (11) +[b06n12:177119] Signal code: (3) +[b06n12:177119] Failing at address: 0x14170000000000a0 +[b06n12:177119] [ 0] [0x1000000504d8] +[b06n12:177119] [ 1] [0x3fffe84511d0] +[b06n12:177119] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n12:177119] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n17:126527] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n17:126527] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n17:126527] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n17:126527] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n17:126527] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n11:152815] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n11:152815] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n11:152815] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n11:152815] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n11:152815] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n11:152815] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n11:152815] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n11:152815] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n11:152815] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n11:152815] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n15:123040] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n15:123040] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n15:123040] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n11:02684] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n12:140120] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n12:140120] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n03:168930] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n03:168930] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n03:168930] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n03:168930] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n03:168930] [13] [0x3fffe14a0dc0] +[a33n09:147790] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n09:147790] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n09:147790] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n09:164363] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n09:164363] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n09:164363] [ 6] [a35n03:135164] *** Process received signal *** +[a35n03:135164] Signal: Segmentation fault (11) +[a35n03:135164] Signal code: (3) +[a35n03:135164] Failing at address: 0x14170000000000a0 +[a35n03:135164] [ 0] [0x1000000504d8] +[a35n03:135164] [ 1] [0x3ffffb3c4d10] +[a35n03:135164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n03:135164] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n03:135164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n03:135164] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n12:11420] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n17:119468] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n17:119468] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n17:119468] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n17:119468] [ 8] [a27n02:49105] *** Process received signal *** +[a27n02:49105] Signal: Segmentation fault (11) +[a27n02:49105] Signal code: (3) +[a27n02:49105] Failing at address: 0x5170000000000a0 +[a27n02:49105] [ 0] [a30n14:163549] *** Process received signal *** +[a30n14:163549] Signal: Segmentation fault (11) +[a30n14:163549] Signal code: (3) +[a30n14:163549] Failing at address: 0x14170000000000a0 +[a30n14:163549] [ 0] [0x1000000504d8] +[a30n14:163549] [ 1] [0x3fffce6113f0] +[a30n14:163549] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n14:163549] [ 3] [b06n15:161756] *** Process received signal *** +[b06n15:161756] Signal: Segmentation fault (11) +[b06n15:161756] Signal code: (3) +[b06n15:161756] Failing at address: 0x14170000000000a0 +[b06n15:161756] [ 0] [0x1000000504d8] +[b06n15:161756] [ 1] [0x3fffc2b7d3c0] +[a29n01:153682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n01:153682] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n01:153682] [ 4] [0x3ffff0014ec0] +[a27n14:21316] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n14:21316] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n14:21316] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n14:21316] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n16:124797] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n16:124797] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n16:124797] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n16:124797] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n16:124797] *** End of error message *** +[0x1000000504d8] +[b07n06:177003] [ 1] [0x3fffe82bb040] +[b07n06:177003] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n06:177003] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n06:177003] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n06:177003] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n17:146798] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n17:146798] *** End of error message *** +[0x1000000504d8] +[b18n03:136667] [ 1] [0x3ffff5f418f0] +[b18n03:136667] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n03:136667] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n03:136667] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n02:123707] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n02:123707] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n02:123707] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n02:123707] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n03:167778] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n03:167778] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n03:167778] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n03:125241] [ 4] [a26n01:65727] *** Process received signal *** +[a26n01:65727] Signal: Segmentation fault (11) +[a26n01:65727] Signal code: (3) +[a26n01:65727] Failing at address: 0x14170000000000a0 +[a26n01:65727] [ 0] [0x1000000504d8] +[a26n01:65727] [ 1] [0x3fffdad26320] +[a26n01:65727] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n01:65727] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n01:65727] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n06:154622] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n06:154622] *** End of error message *** +[b06n05:163963] *** Process received signal *** +[b06n05:163963] Signal: Segmentation fault (11) +[b06n05:163963] Signal code: (3) +[b06n05:163963] Failing at address: 0xb170000000000a0 +[b06n05:163963] [ 0] [0x1000000504d8] +[b06n05:163963] [ 1] [0x3fffeef0e570] +[b06n05:163963] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n05:163963] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n05:163963] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n05:163963] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n05:163963] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n05:163963] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n05:163963] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n05:163963] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n05:163963] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n05:163963] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n05:163963] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n05:163963] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n05:163963] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n05:163963] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n05:162565] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n05:162565] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n03:01352] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n03:01352] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n03:01352] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n03:01352] [10] [a35n11:148674] *** Process received signal *** +[a35n11:148674] Signal: Segmentation fault (11) +[a35n11:148674] Signal code: (3) +[a35n11:148674] Failing at address: 0x5170000000000a0 +[a35n11:148674] [ 0] [0x1000000504d8] +[a35n11:148674] [ 1] [0x3fffd7efea00] +[a35n11:148674] [ 2] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n04:169707] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n04:169707] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n01:164963] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n01:164963] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n01:164963] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n01:164963] [ 9] [0x1000000504d8] +[a26n04:54379] [ 1] [0x3ffff5fa2cd0] +[a26n04:54379] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n04:54379] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n04:54379] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n04:54379] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n04:54379] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n04:54379] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n04:54379] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n04:54379] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n04:54379] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n02:127220] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n02:127220] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n02:127220] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n02:127220] *** End of error message *** +[a28n02:178473] *** Process received signal *** +[a28n02:178473] Signal: Segmentation fault (11) +[a28n02:178473] Signal code: (3) +[a28n02:178473] Failing at address: 0xb170000000000a0 +[a28n02:178473] [ 0] [0x1000000504d8] +[a28n02:178473] [ 1] [0x3fffe080b2c0] +[a28n02:178473] [ 2] [ 0] [0x1000000504d8] +[b06n17:163816] [ 1] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n05:131261] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n05:131261] [ 4] [b15n02:122556] *** Process received signal *** +[b15n02:122556] Signal: Segmentation fault (11) +[b15n02:122556] Signal code: (3) +[b15n02:122556] Failing at address: 0xb170000000000a0 +[b15n02:122556] [ 0] [0x1000000504d8] +[b15n02:122556] [ 1] [b11n10:161717] *** Process received signal *** +[b11n10:161717] Signal: Segmentation fault (11) +[b11n10:161717] Signal code: (3) +[b11n10:161717] Failing at address: 0xb170000000000a0 +[b11n10:161717] [ 0] [b07n09:163516] *** Process received signal *** +[b07n09:163516] Signal: Segmentation fault (11) +[b07n09:163516] Signal code: (3) +[b07n09:163516] Failing at address: 0x14170000000000a0 +[b07n09:163516] [ 0] [0x1000000504d8] +[b07n09:163516] [ 1] [0x3ffff4be7a60] +[b07n09:163516] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n09:163516] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n09:163516] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n09:163516] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n09:163516] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n09:163516] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n09:163516] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n09:163516] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n09:163516] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n09:163516] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n09:163516] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n09:163516] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n09:163516] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n09:163516] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n07:03459] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n07:03459] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n07:03459] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n17:162360] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n17:162360] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n17:162360] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n17:162360] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n17:162360] *** End of error message *** +[b21n04:118345] *** Process received signal *** +[b21n04:118345] Signal: Segmentation fault (11) +[b21n04:118345] Signal code: (3) +[b21n04:118345] Failing at address: 0x14170000000000a0 +[b21n04:118345] [ 0] [0x1000000504d8] +[b21n04:118345] [ 1] [0x3fffdde21410] +[b21n04:118345] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n04:118345] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n04:118345] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n06:130543] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n06:130543] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n06:130543] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n03:153854] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n03:153854] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n03:153854] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n03:153854] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n03:153854] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n03:153854] *** End of error message *** +[0x1000000504d8] +[b18n06:133952] [ 1] [0x3fffd8c95b50] +[b18n06:133952] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n06:133952] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n06:133952] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n01:171784] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n04:13081] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n03:151881] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n03:151881] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n03:151881] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n05:135122] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n05:135122] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n05:135122] [ 6] [b08n09:174687] *** Process received signal *** +[b08n09:174687] Signal: Segmentation fault (11) +[b08n09:174687] Signal code: (3) +[b08n09:174687] Failing at address: 0x14170000000000a0 +[b08n09:174687] [ 0] [0x1000000504d8] +[b08n09:174687] [ 1] [0x3fffe5fde3f0] +[b08n09:174687] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n09:174687] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n09:174687] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n09:174687] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n09:174687] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n09:174687] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n09:174687] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n09:174687] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n09:174687] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n09:174687] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n09:174687] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n09:174687] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n09:174687] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n09:174687] *** End of error message *** +[0x1000000504d8] +[a36n02:147884] [ 1] [0x3fffdcd45230] +[a36n02:147884] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n02:147884] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n02:147884] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n02:147884] [ 5] [a26n05:54534] *** Process received signal *** +[a26n05:54534] Signal: Segmentation fault (11) +[a26n05:54534] Signal code: (3) +[a26n05:54534] Failing at address: 0xb170000000000a0 +[a26n05:54534] [ 0] [0x1000000504d8] +[a26n05:54534] [ 1] [0x3fffdd4562e0] +[a26n05:54534] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n05:54534] [ 3] [a28n08:175804] *** Process received signal *** +[a28n08:175804] Signal: Segmentation fault (11) +[a28n08:175804] Signal code: (3) +[a28n08:175804] Failing at address: 0x5170000000000a0 +[a28n08:175804] [ 0] [0x1000000504d8] +[a28n08:175804] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n11:164995] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n11:164995] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n11:164995] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n11:164995] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n11:164995] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n11:164995] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n11:164995] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n11:164995] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n11:164995] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n11:164995] *** End of error message *** +[ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n08:172426] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n08:172426] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n08:172426] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n04:142256] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n04:142256] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n04:142256] [10] [0x3fffc3c30170] +[a28n01:179262] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n01:179262] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n01:179262] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n01:179262] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n05:130805] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n05:130805] *** End of error message *** +[b21n17:118574] *** Process received signal *** +[b21n17:118574] Signal: Segmentation fault (11) +[b21n17:118574] Signal code: (3) +[b21n17:118574] Failing at address: 0x14170000000000a0 +[b21n17:118574] [ 0] [0x1000000504d8] +[b21n17:118574] [ 1] [0x3fffc6ff9eb0] +[b21n17:118574] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n17:118574] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n18:118680] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n10:164939] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n10:164939] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n10:164939] [ 8] [0x1000000504d8] +[a36n01:141340] [ 1] [0x3fffe257d900] +[a36n01:141340] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n01:141340] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n01:141340] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n06:162920] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n06:162920] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n06:162920] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n06:162920] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n06:162920] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n06:162920] [ 9] [a36n14:152038] *** Process received signal *** +[a36n14:152038] Signal: Segmentation fault (11) +[a36n14:152038] Signal code: (3) +[a36n14:152038] Failing at address: 0xb170000000000a0 +[a36n14:152038] [ 0] [0x1000000504d8] +[a36n14:152038] [ 1] [0x3fffd6d3ed60] +[a36n14:152038] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n14:152038] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n14:152038] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n14:152038] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n14:152038] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n14:152038] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n14:152038] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n14:152038] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n14:152038] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n14:152038] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n14:152038] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n14:152038] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n14:152038] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n14:152038] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n16:121928] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n16:121928] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n16:121928] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n16:121928] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n04:04850] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n04:04850] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n04:04850] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n02:121683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n02:121683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n02:121683] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n01:158306] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n01:158306] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n01:158306] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n01:158306] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n01:158306] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n01:158306] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n01:158306] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n01:158306] [10] [0x3fffd94dda40] +[a28n16:171167] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n16:171167] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n16:171167] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n16:171167] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n16:171167] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n16:171167] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n16:171167] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n16:171167] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n16:171167] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n04:123054] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n04:123054] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n04:123054] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n01:146656] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n01:146656] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n01:146656] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n01:146656] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n01:146656] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n01:146656] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n01:146656] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n01:146656] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n01:146656] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n01:146656] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n01:146656] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n01:146656] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n08:164751] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n08:164751] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n08:164751] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n09:140255] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n09:140255] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n09:140255] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n09:140255] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n09:140255] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n09:140255] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n09:140255] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n09:140255] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n09:140255] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n09:140255] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n09:140255] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n09:140255] *** End of error message *** +[b22n06:133055] *** Process received signal *** +[b22n06:133055] Signal: Segmentation fault (11) +[b22n06:133055] Signal code: (3) +[b22n06:133055] Failing at address: 0x14170000000000a0 +[b22n06:133055] [ 0] [0x1000000504d8] +[b22n06:133055] [ 1] [0x3fffc7d39bc0] +[b22n06:133055] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n06:133055] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n06:133055] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n06:133055] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n06:133055] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n06:133055] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n06:133055] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n06:133055] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n06:133055] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n06:133055] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n06:133055] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n06:133055] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n06:133055] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n06:133055] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n03:121809] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n03:121809] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n01:132063] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n01:132063] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n05:154185] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n05:154185] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n05:154185] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n15:169493] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n15:169493] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n15:169493] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n15:169493] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n15:169493] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n15:169493] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n15:169493] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n15:169493] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n15:169493] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n15:169493] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n15:169493] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n15:169493] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n15:169493] *** End of error message *** +[b23n02:116888] *** Process received signal *** +[b23n02:116888] Signal: Segmentation fault (11) +[b23n02:116888] Signal code: (3) +[b23n02:116888] Failing at address: 0x14170000000000a0 +[b23n02:116888] [ 0] [0x1000000504d8] +[b23n02:116888] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n05:02909] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n05:02909] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n05:02909] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n05:02909] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n05:02909] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n06:171503] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n06:171503] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n06:171503] [ 5] [b15n06:120734] *** Process received signal *** +[b15n06:120734] Signal: Segmentation fault (11) +[b15n06:120734] Signal code: (3) +[b15n06:120734] Failing at address: 0xb170000000000a0 +[b15n06:120734] [ 0] [0x1000000504d8] +[b15n06:120734] [ 1] [0x3fffda6ba1e0] +[b15n06:120734] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n06:120734] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n06:120734] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n06:132106] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n06:132106] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n06:132106] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n06:132106] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n06:132106] [b15n04:133900] *** Process received signal *** +[b15n04:133900] Signal: Segmentation fault (11) +[b15n04:133900] Signal code: (3) +[b15n04:133900] Failing at address: 0xb170000000000a0 +[b15n04:133900] [ 0] [0x1000000504d8] +[b15n04:133900] [ 1] [0x3fffca8e1120] +[b15n04:133900] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n04:133900] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n04:133900] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n04:133900] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n04:133900] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n04:133900] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n04:133900] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n04:133900] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n04:133900] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n04:133900] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n04:133900] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n04:133900] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n04:133900] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n04:133900] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n07:119635] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n07:119635] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n07:119635] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n07:119635] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n07:119635] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n07:119635] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n07:119635] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n18:162421] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n18:162421] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n18:162421] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n18:162421] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n18:162421] *** End of error message *** +[b03n07:03371] [ 0] [0x1000000504d8] +[b03n07:03371] [ 1] [0x3fffff2ec360] +[b03n07:03371] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n07:03371] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n10:02712] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n10:02712] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n10:02712] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n10:02712] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n10:02712] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n10:02712] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n10:02712] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n10:02712] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n05:122272] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n05:122272] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n05:122272] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n05:122272] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n08:121433] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n08:121433] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n08:121433] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n08:121433] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n08:121433] [ 9] [b14n02:139024] *** Process received signal *** +[b14n02:139024] Signal: Segmentation fault (11) +[b14n02:139024] Signal code: (3) +[b14n02:139024] Failing at address: 0xb170000000000a0 +[b14n02:139024] [ 0] [0x1000000504d8] +[b14n02:139024] [ 1] [0x3fffe4a5ac30] +[b14n02:139024] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n02:139024] [ 3] [0x3ffff8566330] +[b14n03:130346] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n03:130346] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n03:130346] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n03:130346] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n03:130346] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n03:130346] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n03:130346] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n06:163677] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n06:163677] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n06:163677] [ 9] [b06n11:160799] *** Process received signal *** +[b06n11:160799] Signal: Segmentation fault (11) +[b06n11:160799] Signal code: (3) +[b06n11:160799] Failing at address: 0x14170000000000a0 +[b06n11:160799] [ 0] [0x1000000504d8] +[b06n11:160799] [ 1] [0x3ffffeb47fa0] +[b06n11:160799] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n11:160799] [ 3] [b10n05:173080] *** Process received signal *** +[b10n05:173080] Signal: Segmentation fault (11) +[b10n05:173080] Signal code: (3) +[b10n05:173080] Failing at address: 0xb170000000000a0 +[b14n01:128005] *** Process received signal *** +[b14n01:128005] Signal: Segmentation fault (11) +[b14n01:128005] Signal code: (3) +[b14n01:128005] Failing at address: 0xb170000000000a0 +[b14n01:128005] [ 0] [0x1000000504d8] +[b14n01:128005] [ 1] [0x3ffff30be100] +[b14n01:128005] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n01:128005] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n05:120332] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n05:120332] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n05:120332] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n12:138737] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n12:138737] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n12:138737] [12] [0x3ffff6f48a70] +[b10n16:164549] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n16:164549] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n16:164549] [ 4] [b14n04:126861] *** Process received signal *** +[b14n04:126861] Signal: Segmentation fault (11) +[b14n04:126861] Signal code: (3) +[b14n04:126861] Failing at address: 0x14170000000000a0 +[b14n04:126861] [ 0] [0x1000000504d8] +[b14n04:126861] [ 1] [0x3fffc98f52f0] +[b14n04:126861] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n04:126861] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n04:126861] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n04:126861] [ 5] [a32n15:139056] *** Process received signal *** +[a32n15:139056] Signal: Segmentation fault (11) +[a32n15:139056] Signal code: (3) +[a32n15:139056] Failing at address: 0xb170000000000a0 +[a32n15:139056] [ 0] [0x1000000504d8] +[a32n15:139056] [ 1] [0x3ffff5288da0] +[a32n15:139056] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n15:139056] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n15:139056] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n15:139056] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n15:139056] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n15:139056] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n15:139056] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n15:139056] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n15:139056] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n15:139056] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n15:139056] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n15:139056] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n15:139056] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n15:139056] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n17:168151] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n17:168151] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n17:168151] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n17:168151] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n17:168151] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n17:168151] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n07:124352] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n09:155757] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n09:155757] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n09:155757] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n09:155757] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n17:176414] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n17:176414] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n17:176414] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n17:176414] [ 9] [0x1000000504d8] +[b19n17:123235] [ 1] [0x3fffdc2f2250] +[b19n17:123235] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n17:123235] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n17:123235] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n17:123235] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n13:10199] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n13:10199] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n13:10199] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n13:10199] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n13:10199] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n13:10199] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n13:10199] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n13:10199] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n13:10199] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n13:10199] *** End of error message *** +[ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n01:165124] [ 6] [a29n06:157789] *** Process received signal *** +[a29n06:157789] Signal: Segmentation fault (11) +[a29n06:157789] Signal code: (3) +[a29n06:157789] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n06:138766] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n06:138766] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n06:138766] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n06:138766] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n06:138766] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n06:161720] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n06:161720] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n06:161720] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n06:161720] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n06:161720] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n11:117475] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n11:117475] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n11:117475] [12] [b08n18:164012] *** Process received signal *** +[b08n18:164012] Signal: Segmentation fault (11) +[b08n18:164012] Signal code: (3) +[b08n18:164012] Failing at address: 0xb170000000000a0 +[b08n18:164012] [ 0] [0x1000000504d8] +[b08n18:164012] [ 1] [0x3fffe3223770] +[b08n18:164012] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n18:164012] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n18:164012] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n18:164012] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n18:164012] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n18:164012] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n18:164012] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n18:164012] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n18:164012] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n18:164012] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n18:164012] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n18:164012] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n18:164012] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n18:164012] *** End of error message *** +[0x3ffff3e7c870] +[b14n05:126782] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n05:126782] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n05:126782] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n05:126782] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n05:126782] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n05:126782] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n05:126782] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n05:126782] [ 9] [a02n07:152083] *** Process received signal *** +[a02n07:152083] Signal: Segmentation fault (11) +[a02n07:152083] Signal code: (3) +[a02n07:152083] Failing at address: 0xb170000000000a0 +[a02n07:152083] [ 0] [0x1000000504d8] +[a02n07:152083] [ 1] [0x3fffcd0b03a0] +[a02n07:152083] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n07:152083] [ 3] [b23n06:117526] Signal: Segmentation fault (11) +[b23n06:117526] Signal code: (3) +[b23n06:117526] Failing at address: 0xb170000000000a0 +[b23n06:117526] [ 0] [0x1000000504d8] +[b23n06:117526] [ 1] [0x3fffdff6ff70] +[b23n06:117526] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n06:117526] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n09:119634] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n09:119634] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n09:119634] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n09:119634] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n03:154295] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n03:154295] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n03:154295] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n03:154295] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n03:154295] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n03:154295] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n03:154295] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n03:154295] [14] [b11n13:153675] *** Process received signal *** +[b11n13:153675] Signal: Segmentation fault (11) +[b11n13:153675] Signal code: (3) +[b11n13:153675] Failing at address: 0x14170000000000a0 +[b11n13:153675] [ 0] [0x1000000504d8] +[b11n13:153675] [ 1] [0x3fffffc91240] +[b07n10:16972] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n10:16972] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n10:16972] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n10:16972] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n16:148188] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n16:148188] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n07:168962] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n07:168962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n07:168962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n07:168962] [ 8] [b22n03:120667] *** Process received signal *** +[b22n03:120667] Signal: Segmentation fault (11) +[b22n03:120667] Signal code: (3) +[b22n03:120667] Failing at address: 0xb170000000000a0 +[b22n03:120667] [ 0] [0x1000000504d8] +[b22n03:120667] [ 1] [0x3fffe8a70b60] +[b22n03:120667] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n03:120667] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n03:46688] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n03:46688] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n11:120580] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n11:120580] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n11:120580] [12] [a29n07:160799] *** Process received signal *** +[a29n07:160799] Signal: Segmentation fault (11) +[a29n07:160799] Signal code: (3) +[a29n07:160799] Failing at address: 0x14170000000000a0 +[a29n07:160799] [ 0] [0x1000000504d8] +[a29n07:160799] [ 1] [0x3fffe86687d0] +[a29n07:160799] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n07:160799] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n07:160799] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n07:160799] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n07:160799] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n07:160799] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n07:160799] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n07:160799] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n07:160799] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n07:160799] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n07:160799] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n07:160799] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n07:160799] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n07:160799] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n04:158636] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n04:158636] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n14:178327] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n14:178327] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n14:178327] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n12:178597] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n12:178597] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n12:178597] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n12:178597] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n13:171109] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n13:171109] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n13:171109] [11] [a35n04:145749] *** Process received signal *** +[a35n04:145749] Signal: Segmentation fault (11) +[a35n04:145749] Signal code: (3) +[a35n04:145749] Failing at address: 0xb170000000000a0 +[a35n04:145749] [ 0] [0x1000000504d8] +[a35n04:145749] [ 1] [0x3fffe991fed0] +[a35n04:145749] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n05:139434] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n05:139434] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n05:139434] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n05:139434] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n05:139434] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n05:139434] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n05:139434] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n05:139434] [11] [b01n11:03985] *** Process received signal *** +[b01n11:03985] Signal: Segmentation fault (11) +[b01n11:03985] Signal code: (3) +[b01n11:03985] Failing at address: 0x14170000000000a0 +[b01n11:03985] [ 0] [0x1000000504d8] +[b01n11:03985] [ 1] [0x3fffddc76830] +[b01n11:03985] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n11:03985] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n10:11489] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n10:11489] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n10:11489] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n10:11489] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n10:11489] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n12:141259] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n12:141259] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n12:141259] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n12:141259] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n12:141259] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n05:123934] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n05:123934] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n05:123934] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n05:123934] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n05:123934] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n05:123934] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n05:123934] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n05:123934] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n05:123934] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n05:123934] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n05:123934] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n05:123934] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n05:123934] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n17:169737] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n17:169737] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n18:179171] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n18:179171] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n18:179171] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n18:179171] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n18:179171] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n17:119977] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n17:119977] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n17:119977] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n17:119977] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n17:119977] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n17:119977] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n16:08386] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n16:08386] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n16:08386] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n16:08386] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n16:08386] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n05:139910] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n05:139910] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n05:139910] [14] [b13n11:134518] *** Process received signal *** +[b13n11:134518] Signal: Segmentation fault (11) +[b13n11:134518] Signal code: (3) +[b13n11:134518] Failing at address: 0xb170000000000a0 +[b13n11:134518] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n04:44756] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n04:44756] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n04:44756] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n12:123558] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n12:123558] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n12:123558] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n12:123558] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n03:06825] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n03:06825] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n03:06825] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n03:06825] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n03:06825] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n03:06825] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n03:06825] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n03:06825] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n14:08398] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n14:08398] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n14:08398] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n13:123517] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n13:123517] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n13:123517] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n13:123517] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n13:123517] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n13:123517] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n13:123517] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n13:123517] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n05:156009] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n05:156009] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n05:156009] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n05:156009] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n16:162629] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n16:162629] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n16:162629] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n16:162629] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n16:162629] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n16:162629] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n16:162629] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n16:162629] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n16:162629] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n16:162629] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n16:162629] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n16:162629] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n13:154235] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n05:174782] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n05:174782] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n05:174782] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n05:174782] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n05:174782] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n05:174782] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n05:174782] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n05:174782] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n05:174782] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n10:169208] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n10:169208] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n10:169208] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n10:169208] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n10:169208] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n10:169208] [10] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n04:03149] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n04:03149] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n04:03149] *** End of error message *** +[b15n14:133966] *** Process received signal *** +[b15n14:133966] Signal: Segmentation fault (11) +[b15n14:133966] Signal code: (3) +[b15n14:133966] Failing at address: 0xb170000000000a0 +[b15n14:133966] [ 0] [0x1000000504d8] +[b15n14:133966] [ 1] [0x3fffc18031e0] +[b15n14:133966] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n14:133966] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n14:133966] [ 4] [b09n08:176680] [ 0] [0x1000000504d8] +[b09n08:176680] [ 1] [0x3ffff5fce0b0] +[b09n08:176680] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n08:176680] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n08:176680] [ 4] [b12n08:141874] *** Process received signal *** +[b12n08:141874] Signal: Segmentation fault (11) +[b12n08:141874] Signal code: (3) +[b12n08:141874] Failing at address: 0xb170000000000a0 +[b12n08:141874] [ 0] [0x1000000504d8] +[b12n08:141874] [ 1] [0x3fffe36363c0] +[b12n08:141874] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n08:141874] [ 3] [a30n01:163391] *** Process received signal *** +[a30n01:163391] Signal: Segmentation fault (11) +[a30n01:163391] Signal code: (3) +[a30n01:163391] Failing at address: 0x14170000000000a0 +[a30n01:163391] [ 0] [0x1000000504d8] +[a30n01:163391] [ 1] [0x3fffc6a4acd0] +[a30n01:163391] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n01:163391] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n16:53798] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n16:53798] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n16:53798] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n16:53798] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n16:53798] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n16:53798] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n16:53798] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n10:118386] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n10:118386] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n10:118386] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n10:118386] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n10:118386] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n10:118386] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n10:118386] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n13:123055] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n13:123055] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n13:123055] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n12:160964] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n12:160964] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n12:160964] [ 9] [b11n15:166119] *** Process received signal *** +[b11n15:166119] Signal: Segmentation fault (11) +[b11n15:166119] Signal code: (3) +[b11n15:166119] Failing at address: 0x14170000000000a0 +[b11n15:166119] [ 0] [0x1000000504d8] +[b11n15:166119] [ 1] [0x3fffc6a31d70] +[b11n15:166119] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n15:166119] [ 3] [a33n10:140416] *** Process received signal *** +[a33n10:140416] Signal: Segmentation fault (11) +[a33n10:140416] Signal code: (3) +[a33n10:140416] Failing at address: 0x14170000000000a0 +[a33n10:140416] [ 0] [a28n06:179451] *** Process received signal *** +[a28n06:179451] Signal: Segmentation fault (11) +[a28n06:179451] Signal code: (3) +[a28n06:179451] Failing at address: 0xb170000000000a0 +[a28n06:179451] [ 0] [0x1000000504d8] +[a28n06:179451] [ 1] [0x3fffcc4b6400] +[a28n06:179451] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n06:179451] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n06:179451] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n06:179451] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n06:179451] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n06:179451] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n06:179451] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n06:179451] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n06:179451] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n06:179451] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n06:179451] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n06:179451] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n06:179451] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n06:179451] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n01:170765] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n01:170765] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n01:170765] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n01:170765] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n01:170765] *** End of error message *** +[b14n11:130804] *** Process received signal *** +[b14n11:130804] Signal: Segmentation fault (11) +[b14n11:130804] Signal code: (3) +[b14n11:130804] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n09:12247] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n09:12247] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n09:12247] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n18:128539] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n18:128539] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n18:128539] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n18:128539] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n18:128539] [ 8] [a34n12:145497] *** Process received signal *** +[a34n12:145497] Signal: Segmentation fault (11) +[a34n12:145497] Signal code: (3) +[a34n12:145497] Failing at address: 0x14170000000000a0 +[a34n12:145497] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n18:123412] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n18:123412] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n18:123412] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n18:123412] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n08:159022] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n08:159022] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n10:131275] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n10:131275] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n10:131275] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n10:131275] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n10:131275] [13] [b23n08:116905] *** Process received signal *** +[b23n08:116905] Signal: Segmentation fault (11) +[b23n08:116905] Signal code: (3) +[b23n08:116905] Failing at address: 0x14170000000000a0 +[b23n08:116905] [ 0] [0x1000000504d8] +[b23n08:116905] [ 1] [0x3fffff679930] +[b23n08:116905] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n08:116905] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n08:116905] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n08:116905] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n08:116905] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n08:116905] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n08:116905] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n08:116905] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n08:116905] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n08:116905] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n08:116905] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n08:116905] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n08:116905] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n08:116905] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n05:04521] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n05:04521] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n05:04521] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n05:04521] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n05:04521] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n05:120230] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n05:120230] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n05:120230] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n05:120230] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n16:154184] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n16:154184] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n16:154184] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n16:154184] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n15:120132] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n15:120132] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n15:120132] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n15:120132] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n13:133624] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n13:133624] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n13:133624] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n13:133624] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n03:172648] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n03:172648] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n03:172648] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n18:149117] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n18:149117] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n18:149117] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n10:126613] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n10:126613] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n17:138616] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n17:138616] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n17:138616] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n17:138616] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n14:126464] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n14:126464] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n14:126464] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n14:126464] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n14:126464] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n01:96751] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n01:96751] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n01:96751] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n01:96751] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n01:96751] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n06:119114] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n06:119114] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n15:122082] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n15:122082] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n15:122082] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n15:122082] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n15:122082] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n08:127445] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n08:127445] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n11:156154] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n11:156154] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n01:124278] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n01:124278] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n01:124278] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n01:124278] [ 7] [b04n08:176174] *** Process received signal *** +[b04n08:176174] Signal: Segmentation fault (11) +[b04n08:176174] Signal code: (3) +[b04n08:176174] Failing at address: 0x14170000000000a0 +[b04n08:176174] [ 0] [0x1000000504d8] +[b04n08:176174] [ 1] [a30n18:10724] *** Process received signal *** +[b07n16:166496] *** Process received signal *** +[b07n16:166496] Signal: Segmentation fault (11) +[b07n16:166496] Signal code: (3) +[b07n16:166496] Failing at address: 0xb170000000000a0 +[b07n16:166496] [ 0] [0x1000000504d8] +[b07n16:166496] [ 1] [0x3fffd80aa440] +[b07n16:166496] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n16:166496] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n16:166496] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n16:166496] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n16:166496] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n16:166496] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n16:166496] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n16:166496] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n16:166496] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n16:166496] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n16:166496] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n16:166496] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n16:166496] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n16:166496] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n02:122178] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n02:122178] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n02:122178] [13] [b02n10:01253] *** Process received signal *** +[b02n10:01253] Signal: Segmentation fault (11) +[b02n10:01253] Signal code: (3) +[b02n10:01253] Failing at address: 0x14170000000000a0 +[b02n10:01253] [ 0] [0x1000000504d8] +[b02n10:01253] [ 1] [0x3ffff42b0b70] +[b02n10:01253] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n10:01253] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n10:01253] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n10:01253] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n10:01253] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n10:01253] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n10:01253] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n10:01253] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n10:01253] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n10:01253] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n10:01253] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n10:01253] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n10:01253] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n10:01253] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n15:139746] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n15:139746] [ 6] [b13n06:133802] *** Process received signal *** +[b13n06:133802] Signal: Segmentation fault (11) +[b13n06:133802] Signal code: (3) +[b13n06:133802] Failing at address: 0xb170000000000a0 +[b13n06:133802] [ 0] [0x1000000504d8] +[b13n06:133802] [ 1] [0x3fffe66378e0] +[b13n06:133802] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n06:133802] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n06:133802] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n06:133802] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n06:133802] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n06:133802] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n06:133802] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n06:133802] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n06:133802] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n06:133802] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n06:133802] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n06:133802] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n06:133802] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n06:133802] *** End of error message *** +[b06n09:168245] *** Process received signal *** +[b06n09:168245] Signal: Segmentation fault (11) +[b06n09:168245] Signal code: (3) +[b06n09:168245] Failing at address: 0x5170000000000a0 +[b06n09:168245] [ 0] [0x1000000504d8] +[b06n09:168245] [ 1] [0x3ffff7082d30] +[b06n09:168245] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n09:168245] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n14:51451] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n14:51451] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n14:51451] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n14:51451] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n14:51451] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n14:51451] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n14:51451] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n14:51451] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n14:51451] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n14:51451] *** End of error message *** +[b09n13:164688] Signal: Segmentation fault (11) +[b09n13:164688] Signal code: (3) +[b09n13:164688] Failing at address: 0xb170000000000a0 +[b09n13:164688] [ 0] [0x1000000504d8] +[b09n13:164688] [ 1] [0x3fffc5cdb170] +[b09n13:164688] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n13:164688] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n13:164688] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n13:164688] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n13:164688] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n13:164688] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n13:164688] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n13:164688] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n13:164688] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n13:164688] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n13:164688] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n13:164688] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n13:164688] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n13:164688] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n11:176582] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n11:176582] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n11:176582] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n11:176582] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n11:176582] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n11:176582] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n11:176582] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n11:176582] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n11:176582] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n11:176582] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n11:176582] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n11:176582] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n03:09642] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n03:09642] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n03:09642] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n03:09642] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n18:169359] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n18:169359] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n18:169359] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n18:169359] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n18:169359] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n18:169359] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n18:169359] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n18:169359] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n18:169359] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n18:169359] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n18:169359] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n10:10922] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n10:10922] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n10:10922] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n10:10922] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n10:10922] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n05:173317] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n05:173317] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n05:173317] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n05:173317] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n05:173317] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n05:173317] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n10:120979] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n10:120979] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n10:120979] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n10:120979] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n10:120979] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n10:120979] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n10:120979] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n10:155396] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n10:155396] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n10:155396] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n10:155396] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n10:155396] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n10:155396] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n10:169683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n10:169683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n10:169683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n14:152748] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n14:152748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n14:152748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n14:152748] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n14:152748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n01:121742] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n01:121742] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n07:172877] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n07:172877] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n07:172877] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n07:172877] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n07:172877] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n07:172877] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n07:172877] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n07:172877] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n07:172877] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n07:172877] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n15:164413] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n15:164413] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n15:164413] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n15:164413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n15:164413] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n15:164413] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n15:164413] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n15:164413] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n15:164413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n15:164413] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n15:164413] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n15:164413] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n07:165090] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n07:165090] [ 5] [a02n09:160372] *** Process received signal *** +[a02n09:160372] Signal: Segmentation fault (11) +[a02n09:160372] Signal code: (3) +[a02n09:160372] Failing at address: 0x14170000000000a0 +[a02n09:160372] [ 0] [0x1000000504d8] +[a02n09:160372] [ 1] [0x3fffea522f10] +[a02n09:160372] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n09:160372] [ 3] [a26n02:74427] *** Process received signal *** +[a26n02:74427] Signal: Segmentation fault (11) +[a26n02:74427] Signal code: (3) +[a26n02:74427] Failing at address: 0xb170000000000a0 +[a26n02:74427] [ 0] [0x1000000504d8] +[a26n02:74427] [ 1] [0x3fffd0ab74c0] +[a26n02:74427] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n02:74427] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n02:74427] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n02:74427] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n02:74427] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n02:74427] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n02:74427] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n02:74427] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n02:74427] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n02:125266] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n02:125266] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n02:125266] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n02:125266] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n02:125266] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n02:125266] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n02:125266] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n02:125266] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n10:135852] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n05:149466] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n05:149466] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n05:149466] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n05:149466] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n05:149466] [10] [b05n04:06835] *** Process received signal *** +[b05n04:06835] Signal: Segmentation fault (11) +[b05n04:06835] Signal code: (3) +[b05n04:06835] Failing at address: 0x5170000000000a0 +[b05n04:06835] [ 0] [0x1000000504d8] +[b05n04:06835] [ 1] [0x3fffec201c30] +[b05n04:06835] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n04:06835] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n04:06835] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n04:06835] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n04:06835] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n04:06835] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n04:06835] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n04:06835] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n04:06835] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n04:06835] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n04:06835] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n04:06835] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n04:06835] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n04:06835] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n17:122355] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n17:122355] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n17:122355] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n17:122355] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n17:122355] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n17:122355] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n12:122872] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n12:122872] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n12:122872] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n12:122872] [14] [a31n02:158417] *** Process received signal *** +[a31n02:158417] Signal: Segmentation fault (11) +[a31n02:158417] Signal code: (3) +[a31n02:158417] Failing at address: 0xb170000000000a0 +[a31n02:158417] [ 0] [0x1000000504d8] +[a31n02:158417] [ 1] [0x3fffd28ce900] +[a31n02:158417] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n02:158417] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n02:158417] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n02:158417] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n02:158417] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n02:158417] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n02:158417] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n02:158417] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n02:158417] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n02:158417] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n02:158417] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n02:158417] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n02:158417] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n02:158417] *** End of error message *** +[b06n03:164499] *** Process received signal *** +[b06n03:164499] Signal: Segmentation fault (11) +[b06n03:164499] Signal code: (3) +[b06n03:164499] Failing at address: 0x14170000000000a0 +[b06n03:164499] [ 0] [0x1000000504d8] +[b06n03:164499] [ 1] [0x3fffeed1c5d0] +[b06n03:164499] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n03:164499] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n03:164499] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n03:164499] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n03:164499] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n03:164499] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n03:164499] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n03:164499] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n03:164499] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n03:164499] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n03:164499] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n03:164499] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n03:164499] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n03:164499] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n07:04683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n07:04683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n07:04683] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n07:04683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n07:04683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n07:04683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n07:04683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n07:04683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n07:04683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n07:04683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n07:04683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n07:04683] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n03:135036] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n03:135036] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n03:135036] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n03:135036] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n04:130749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n04:130749] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n04:130749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n04:130749] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n04:130749] [13] [0x1000000504d8] +[b15n10:122756] [ 1] [0x3fffec7114f0] +[b15n10:122756] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n10:122756] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n10:122756] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n13:155851] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n13:155851] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n13:155851] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n13:155851] [10] [b07n02:174056] *** Process received signal *** +[b07n02:174056] Signal: Segmentation fault (11) +[b07n02:174056] Signal code: (3) +[b07n02:174056] Failing at address: 0xb170000000000a0 +[b07n02:174056] [ 0] [0x1000000504d8] +[b07n02:174056] [ 1] [0x3fffe602a740] +[b07n02:174056] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n02:174056] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n02:174056] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n02:174056] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n02:174056] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n02:174056] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n02:174056] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n02:174056] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n02:174056] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n02:174056] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n02:174056] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n02:174056] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n02:174056] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n02:174056] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n04:129406] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n04:129406] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n04:129406] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n04:129406] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n04:129406] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n04:129406] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n04:129406] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n04:129406] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n04:129406] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n04:129406] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n17:163178] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n17:163178] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n17:163178] *** End of error message *** +[b05n14:177513] *** Process received signal *** +[b05n14:177513] Signal: Segmentation fault (11) +[b05n14:177513] Signal code: (3) +[b05n14:177513] Failing at address: 0x14170000000000a0 +[b05n14:177513] [ 0] [0x1000000504d8] +[b05n14:177513] [ 1] [0x3ffff4baecb0] +[b05n14:177513] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n14:177513] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n14:177513] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n12:160189] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n12:160189] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n12:160189] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n12:160189] [14] [b05n09:106787] *** Process received signal *** +[b05n09:106787] Signal: Segmentation fault (11) +[b05n09:106787] Signal code: (3) +[b05n09:106787] Failing at address: 0xb170000000000a0 +[b05n09:106787] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n09:170658] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n09:170658] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n07:120698] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n07:120698] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n07:120698] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n07:120698] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n07:120698] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n07:120698] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n07:120698] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n07:120698] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n07:120698] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n07:120698] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n07:45305] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n07:45305] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n07:45305] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n07:45305] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n05:176911] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n05:176911] [ 5] [b18n16:124960] *** Process received signal *** +[b18n16:124960] Signal: Segmentation fault (11) +[b18n16:124960] Signal code: (3) +[b18n16:124960] Failing at address: 0x14170000000000a0 +[b18n16:124960] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n02:125370] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n02:125370] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n02:125370] [11] [b17n09:128493] *** Process received signal *** +[b17n09:128493] Signal: Segmentation fault (11) +[b17n09:128493] Signal code: (3) +[b17n09:128493] Failing at address: 0x14170000000000a0 +[b17n09:128493] [ 0] [0x1000000504d8] +[b17n09:128493] [ 1] [0x3fffd06544b0] +[b17n09:128493] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n08:118554] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n08:118554] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n08:118554] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n08:118554] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n08:118554] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n08:118554] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n08:118554] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n08:118554] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n08:146119] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n08:146119] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n08:146119] [13] [a31n14:165662] *** Process received signal *** +[a31n14:165662] Signal: Segmentation fault (11) +[a31n14:165662] Signal code: (3) +[a31n14:165662] Failing at address: 0x14170000000000a0 +[a31n14:165662] [ 0] [0x1000000504d8] +[a31n14:165662] [ 1] [0x3fffdc4db710] +[a31n14:165662] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n14:165662] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n14:165662] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n14:165662] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n14:165662] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n14:165662] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n14:165662] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n14:165662] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n14:165662] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n14:165662] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n14:165662] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n14:165662] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n14:165662] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n14:165662] *** End of error message *** +[b04n07:02840] [ 0] [0x1000000504d8] +[b04n07:02840] [ 1] [0x3fffc16afd20] +[b04n07:02840] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n07:02840] [ 3] [b17n14:126119] *** Process received signal *** +[b17n14:126119] Signal: Segmentation fault (11) +[b17n14:126119] Signal code: (3) +[b17n14:126119] Failing at address: 0x14170000000000a0 +[b17n14:126119] [ 0] [0x1000000504d8] +[b17n14:126119] [ 1] [0x3fffe4502690] +[b17n14:126119] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n02:139631] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n02:139631] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n02:139631] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n02:139631] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n02:139631] *** End of error message *** +[b18n04:136996] *** Process received signal *** +[b18n04:136996] Signal: Segmentation fault (11) +[b18n04:136996] Signal code: (3) +[b18n04:136996] Failing at address: 0xb170000000000a0 +[b18n04:136996] [ 0] [0x1000000504d8] +[b18n04:136996] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n04:156414] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n04:156414] [ 6] [a31n07:156487] *** Process received signal *** +[a31n07:156487] Signal: Segmentation fault (11) +[a31n07:156487] Signal code: (3) +[a31n07:156487] Failing at address: 0xb170000000000a0 +[a31n07:156487] [ 0] [0x1000000504d8] +[a31n07:156487] [ 1] [0x3fffdac732a0] +[a31n07:156487] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n07:156487] [ 3] [b13n03:126187] Signal: Segmentation fault (11) +[b13n03:126187] Signal code: (3) +[b13n03:126187] Failing at address: 0xb170000000000a0 +[b13n03:126187] [ 0] [0x1000000504d8] +[b13n03:126187] [ 1] [0x3ffff7f535e0] +[b13n03:126187] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n03:126187] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n15:165473] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n15:165473] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n15:165473] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n10:176260] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n10:176260] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n10:176260] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n10:176260] *** End of error message *** +[a26n03:68832] *** Process received signal *** +[a26n03:68832] Signal: Segmentation fault (11) +[a26n03:68832] Signal code: (3) +[a26n03:68832] Failing at address: 0x14170000000000a0 +[a26n03:68832] [ 0] [0x1000000504d8] +[a26n03:68832] [ 1] [0x3fffe33967d0] +[a26n03:68832] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n03:68832] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n03:147536] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n03:147536] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n03:147536] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n03:147536] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n03:147536] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n03:147536] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n03:147536] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n03:147536] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n03:147536] [14] [0x3ffffec2c430] +[a34n13:137745] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n13:137745] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n13:137745] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n13:137745] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n13:137745] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n13:137745] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n13:137745] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n13:137745] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n13:137745] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n13:137745] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n13:137745] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n13:137745] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n13:137745] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n13:137745] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n11:134749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n11:134749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n11:134749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n12:145263] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n12:145263] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n12:145263] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n12:145263] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n12:145263] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n12:145263] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n12:145263] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n12:145263] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n12:145263] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n12:145263] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n12:145263] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n08:165281] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n08:165281] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n08:165281] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n08:165281] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n18:123867] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n18:123867] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n18:123867] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n18:123867] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n18:123867] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n18:123867] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n18:123867] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n18:123867] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n18:123867] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n14:122443] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n14:122443] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n14:122443] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n14:122443] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n03:119829] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n03:119829] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n03:119829] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n03:119829] [13] [b17n08:124119] *** Process received signal *** +[b17n08:124119] Signal: Segmentation fault (11) +[b17n08:124119] Signal code: (3) +[b17n08:124119] Failing at address: 0x5170000000000a0 +[b17n08:124119] [ 0] [0x1000000504d8] +[b17n08:124119] [ 1] [0x3fffdb4c7300] +[b17n08:124119] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n08:124119] [ 3] [0x1000000504d8] +[b18n11:128953] [ 1] [0x3fffc3df89f0] +[b18n11:128953] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n11:128953] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n11:128953] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n01:123571] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n01:123571] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n01:123571] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n01:123571] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n05:173822] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n05:173822] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n05:173822] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n05:173822] [ 9] [0x1000000504d8] +[b18n07:125450] [ 1] [0x3fffeb3de4a0] +[b18n07:125450] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n07:125450] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n07:125450] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n07:125450] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n07:163253] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n07:163253] [10] [b12n16:129236] Signal: Segmentation fault (11) +[b12n16:129236] Signal code: (3) +[b12n16:129236] Failing at address: 0x14170000000000a0 +[b12n16:129236] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n13:165388] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n13:165388] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n13:165388] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n06:148310] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n06:148310] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n06:148310] [11] [b17n04:136608] *** Process received signal *** +[b17n04:136608] Signal: Segmentation fault (11) +[b17n04:136608] Signal code: (3) +[b17n04:136608] Failing at address: 0x14170000000000a0 +[b17n04:136608] [ 0] [0x1000000504d8] +[b17n04:136608] [ 1] [b20n03:124682] *** Process received signal *** +[b20n03:124682] Signal: Segmentation fault (11) +[b20n03:124682] Signal code: (3) +[b20n03:124682] Failing at address: 0x14170000000000a0 +[b20n03:124682] [ 0] [0x1000000504d8] +[b20n03:124682] [ 1] [0x3ffff5406ad0] +[b20n03:124682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n03:124682] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n03:155729] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n03:155729] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n03:155729] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n03:155729] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n03:155729] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n03:155729] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n03:155729] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n03:155729] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n03:155729] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n03:155729] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n03:155729] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n12:120401] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n12:120401] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n12:120401] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n12:120401] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n12:120401] [14] [b11n07:156140] [ 0] [0x1000000504d8] +[b11n07:156140] [ 1] [0x3fffd52110f0] +[b11n07:156140] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n07:156140] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n07:156140] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n12:168644] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n12:168644] *** End of error message *** +[a26n13:54242] [ 1] [0x3fffe6011750] +[a26n13:54242] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n13:54242] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n13:54242] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n14:177225] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n14:177225] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n14:177225] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n14:177225] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n14:177225] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n14:177225] [14] [b01n02:01430] *** Process received signal *** +[b01n02:01430] Signal: Segmentation fault (11) +[b01n02:01430] Signal code: (3) +[b01n02:01430] Failing at address: 0x14170000000000a0 +[b01n02:01430] [ 0] [0x1000000504d8] +[b01n02:01430] [ 1] [0x3ffff44d11a0] +[b01n02:01430] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n02:01430] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n02:01430] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n02:01430] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n02:01430] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n02:01430] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n02:01430] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n02:01430] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n02:01430] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n02:01430] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n02:01430] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n02:01430] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n02:01430] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n02:01430] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n18:121304] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n18:121304] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n18:121304] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n18:121304] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n18:121304] *** End of error message *** +[ 0] [0x1000000504d8] +[b03n05:05607] [ 1] [0x3fffc982f870] +[b03n05:05607] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n05:05607] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n05:05607] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n05:05607] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n05:05607] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n05:05607] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n05:05607] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n05:05607] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n05:05607] [10] [0x1000000504d8] +[a27n05:53373] [ 1] [0x3fffc4b07060] +[a27n05:53373] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n05:53373] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n05:53373] [ 4] [b09n04:177598] *** Process received signal *** +[b09n04:177598] Signal: Segmentation fault (11) +[b09n04:177598] Signal code: (3) +[b09n04:177598] Failing at address: 0x14170000000000a0 +[b09n04:177598] [ 0] [0x1000000504d8] +[b09n04:177598] [ 1] [0x3ffff63df850] +[b09n04:177598] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n04:177598] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n12:177496] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n12:177496] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n12:177496] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n12:177496] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n12:177496] [ 9] [b02n09:166573] *** Process received signal *** +[b02n09:166573] Signal: Segmentation fault (11) +[b02n09:166573] Signal code: (3) +[b02n09:166573] Failing at address: 0x14170000000000a0 +[b02n09:166573] [ 0] [0x1000000504d8] +[b02n09:166573] [ 1] [0x3fffe94147f0] +[b02n09:166573] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n09:166573] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n04:117355] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n04:117355] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n04:117355] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n04:117355] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n04:117355] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n09:143898] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n09:143898] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n09:143898] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n09:143898] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n09:143898] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n09:143898] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n09:143898] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n09:143898] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n09:143898] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n09:143898] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n14:139846] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n14:139846] *** End of error message *** +[0x3fffd874e4c0] +[b22n14:116410] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n14:116410] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n14:116410] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n14:116410] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n14:116410] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n16:130375] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n16:130375] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n16:130375] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n16:130375] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n05:121826] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n05:121826] [10] [b13n08:144855] *** Process received signal *** +[b13n08:144855] Signal: Segmentation fault (11) +[b13n08:144855] Signal code: (3) +[b13n08:144855] Failing at address: 0xb170000000000a0 +[b13n08:144855] [ 0] [0x1000000504d8] +[b13n08:144855] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n10:126193] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n10:126193] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n10:126193] [ 8] [a32n16:147287] *** Process received signal *** +[a32n16:147287] Signal: Segmentation fault (11) +[a32n16:147287] Signal code: (3) +[a32n16:147287] Failing at address: 0x14170000000000a0 +[a32n16:147287] [ 0] [0x1000000504d8] +[a32n16:147287] [ 1] [0x3fffe234e400] +[a32n16:147287] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n12:119806] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n12:119806] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n12:119806] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n12:119806] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n07:149479] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n07:149479] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n07:149479] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n07:149479] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n07:149479] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n07:149479] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n07:149479] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n07:149479] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n07:149479] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n07:149479] *** End of error message *** +[0x3fffff545980] +[b02n18:03312] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n18:03312] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n18:03312] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n18:03312] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n18:03312] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n14:120638] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n14:120638] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n14:120638] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n08:138174] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n08:138174] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n08:138174] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n04:168491] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n04:168491] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n04:168491] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n04:168491] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n04:168491] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n04:168491] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n04:168491] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n04:168491] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n13:122188] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n13:122188] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n13:122188] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n13:122188] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n13:177637] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n13:177637] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n17:125808] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n17:125808] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n18:122553] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n18:122553] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n16:168049] *** End of error message *** +[b09n07:173041] *** Process received signal *** +[b09n07:173041] Signal: Segmentation fault (11) +[b09n07:173041] Signal code: (3) +[b09n07:173041] Failing at address: 0xb170000000000a0 +[b09n07:173041] [ 0] [0x1000000504d8] +[b09n07:173041] [ 1] [0x3fffe2fd2640] +[b09n07:173041] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n07:173041] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n07:173041] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n07:173041] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n07:173041] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n07:173041] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n07:173041] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n07:173041] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n07:173041] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n07:173041] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n07:173041] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n07:173041] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n07:173041] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n07:173041] *** End of error message *** +[0x3fffe9ee01a0] +[a35n13:138091] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n13:138091] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n13:138091] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n13:138091] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n12:180117] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n12:180117] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n12:180117] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n12:180117] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n12:180117] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n11:124445] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n11:124445] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n11:124445] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n11:124445] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n11:124445] *** End of error message *** +[a27n15:32401] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n10:124923] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n10:124923] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n10:124923] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n10:124923] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n10:124923] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n10:124923] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n10:124923] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n10:124923] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n10:124923] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n10:124923] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n10:124923] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n14:136137] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n07:174561] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n07:174561] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n07:174561] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n07:174561] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n07:174561] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n10:141233] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n10:141233] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n10:141233] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n15:120057] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n15:120057] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n15:120057] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n15:120057] [12] [b05n01:178524] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n01:178524] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n01:178524] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n01:178524] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n01:178524] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n01:178524] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n01:178524] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n01:178524] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n01:178524] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n01:178524] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n01:178524] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n01:178524] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n12:164381] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n01:163024] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n02:177548] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n02:177548] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n02:177548] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n02:177548] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n02:177548] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n02:177548] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n02:177548] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n15:151252] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n15:151252] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n15:151252] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n15:151252] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n15:151252] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n15:151252] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n15:151252] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n09:173060] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n09:173060] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n09:173060] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n09:173060] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n09:173060] [0x3fffc39ae470] +[b04n18:176809] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n18:176809] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n18:176809] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n18:176809] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n18:176809] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n15:119252] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n18:173979] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n14:150133] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n14:150133] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n14:150133] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n14:150133] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n10:134732] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n10:134732] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n10:134732] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n10:134732] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n05:173850] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n05:173850] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n05:173850] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n05:173850] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n05:173850] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n05:173850] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n05:173850] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n05:173850] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n05:173850] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n05:173850] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n05:173850] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n05:173850] *** End of error message *** +[0x3fffe9e3a4c0] +[b23n10:126792] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n10:126792] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n10:126792] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n10:126792] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n13:117510] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n13:117510] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n13:117510] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n15:148167] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n15:148167] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n15:148167] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n17:168159] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n17:168159] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n17:168159] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n13:166925] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n13:166925] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n13:166925] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n13:166925] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n13:166925] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n13:166925] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n02:119534] [ 6] [0x1000000504d8] +[a31n03:158655] [ 1] [0x3fffe415ab50] +[a31n03:158655] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n03:158655] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n03:158655] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n03:158655] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n11:163259] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n11:163259] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n11:163259] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n04:153966] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n04:153966] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n04:153966] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n04:153966] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n12:178718] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n12:178718] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n12:178718] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n14:119488] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n14:119488] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n09:120692] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n09:120692] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n09:120692] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n09:120692] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n09:120692] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n09:120692] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n09:120692] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n09:120692] *** End of error message *** +[b20n04:126344] *** Process received signal *** +[b20n04:126344] Signal: Segmentation fault (11) +[b20n04:126344] Signal code: (3) +[b20n04:126344] Failing at address: 0xb170000000000a0 +[b20n04:126344] [ 0] [0x1000000504d8] +[b20n04:126344] [ 1] [0x3fffde775170] +[b20n04:126344] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n04:126344] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n16:173549] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n16:173549] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n16:173549] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n16:173549] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n05:125614] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n18:164797] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n18:164797] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n18:164797] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n18:164797] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n18:08427] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n18:08427] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n06:164013] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n06:164013] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n06:164013] [ 8] [a31n11:153415] [ 0] [0x1000000504d8] +[a31n11:153415] [ 1] [0x3fffe2632c10] +[a31n11:153415] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n11:153415] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n13:168162] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n13:168162] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n13:168162] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n13:168162] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n04:179173] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n04:179173] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n04:179173] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n04:179173] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n04:179173] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n04:179173] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n04:179173] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n10:173681] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n10:173681] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n10:173681] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n16:149063] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n16:149063] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n16:149063] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n16:149063] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n16:149063] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n16:149063] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n11:162472] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n11:162472] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n11:162472] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n11:124435] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n11:124435] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n11:124435] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n11:124435] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n12:120131] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n14:133062] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n14:133062] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n14:133062] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n14:133062] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n14:133062] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n09:161232] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n09:161232] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n09:161232] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n09:161232] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n09:161232] [ 8] [b19n15:124800] *** Process received signal *** +[b19n15:124800] Signal: Segmentation fault (11) +[b19n15:124800] Signal code: (3) +[b19n15:124800] Failing at address: 0xb170000000000a0 +[b19n15:124800] [ 0] [0x1000000504d8] +[b19n15:124800] [ 1] [0x1000000504d8] +[a33n02:167598] [ 1] [0x3fffc8586010] +[a33n02:167598] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n02:167598] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n02:167598] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n13:123888] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n18:140382] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n18:140382] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n18:140382] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n18:140382] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n18:140382] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n18:140382] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n18:140382] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n04:166206] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n04:166206] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n04:166206] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n04:166206] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n04:166206] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n05:164943] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n05:164943] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n05:164943] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n05:164943] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n05:164943] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n05:164943] [14] [b14n06:127275] *** Process received signal *** +[b14n06:127275] Signal: Segmentation fault (11) +[b14n06:127275] Signal code: (3) +[b14n06:127275] Failing at address: 0xb170000000000a0 +[b14n06:127275] [ 0] [0x1000000504d8] +[b14n06:127275] [ 1] [0x3ffffc413a90] +[b14n06:127275] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n06:127275] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n06:127275] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n10:162611] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n10:162611] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n10:162611] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n10:162611] [ 7] [0x3ffffb2dbc70] +[b08n17:171199] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n17:171199] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n17:171199] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n17:171199] [ 5] [0x3ffffdcaa830] +[a27n08:54627] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n08:54627] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n08:54627] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n08:54627] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n08:54627] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n17:129932] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n17:129932] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n17:129932] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n17:129932] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n17:129932] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n17:129932] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n09:127999] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n09:127999] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n09:127999] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n09:127999] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n17:124167] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n17:124167] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n17:124167] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n17:124167] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n17:124167] [14] [b13n16:128439] *** Process received signal *** +[b13n16:128439] Signal: Segmentation fault (11) +[b13n16:128439] Signal code: (3) +[b13n16:128439] Failing at address: 0x14170000000000a0 +[b13n16:128439] [ 0] [0x1000000504d8] +[b13n16:128439] [ 1] [0x3fffe32b89f0] +[b13n16:128439] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n16:128439] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n09:137774] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n09:137774] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n09:137774] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n09:137774] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n09:137774] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n09:137774] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n09:137774] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n09:137774] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n09:137774] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n09:137774] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n18:131585] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n18:131585] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n18:131585] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n18:131585] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n18:131585] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n18:131585] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n18:131585] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n08:131378] *** End of error message *** +[a27n04:52707] Signal: Segmentation fault (11) +[a27n04:52707] Signal code: (3) +[a27n04:52707] Failing at address: 0x14170000000000a0 +[a27n04:52707] [ 0] [0x1000000504d8] +[a27n04:52707] [ 1] [0x3fffcea7bf00] +[a27n04:52707] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n04:52707] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n04:52707] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n04:52707] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n04:52707] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n07:133373] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n07:133373] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n07:133373] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n07:133373] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n14:176070] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n14:176070] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n14:176070] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n16:165920] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n16:165920] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n16:165920] [ 8] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n02:176050] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n02:176050] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n02:176050] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n04:41670] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n04:41670] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n04:41670] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n08:76223] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n08:76223] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n08:76223] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n08:76223] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n08:116172] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n08:116172] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n08:116172] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n16:127353] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n16:127353] [ 6] [a29n18:07421] *** Process received signal *** +[a29n18:07421] Signal: Segmentation fault (11) +[a29n18:07421] Signal code: (3) +[a29n18:07421] Failing at address: 0x14170000000000a0 +[a29n18:07421] [ 0] [0x1000000504d8] +[a29n18:07421] [ 1] [0x3fffd1555a80] +[a29n18:07421] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n12:132100] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n12:132100] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n12:132100] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n12:132100] [ 8] [a02n18:21773] *** Process received signal *** +[a02n18:21773] Signal: Segmentation fault (11) +[a02n18:21773] Signal code: (3) +[a02n18:21773] Failing at address: 0x14170000000000a0 +[a02n18:21773] [ 0] [0x1000000504d8] +[a02n18:21773] [ 1] [0x3fffc9fdd350] +[a02n18:21773] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n18:21773] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n18:21773] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n18:21773] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n18:21773] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n18:21773] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n18:21773] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n18:21773] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n18:21773] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n18:21773] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n18:21773] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n18:21773] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n18:21773] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n18:21773] *** End of error message *** +[0x1000000504d8] +[a31n06:167367] [ 1] [0x3fffe3d6e0e0] +[a31n06:167367] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n06:167367] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n06:167367] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n06:167367] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n06:167367] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n18:69156] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n18:69156] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n18:69156] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n18:69156] [14] [b19n03:123615] *** Process received signal *** +[b19n03:123615] Signal: Segmentation fault (11) +[b19n03:123615] Signal code: (3) +[b19n03:123615] Failing at address: 0xb170000000000a0 +[b19n03:123615] [ 0] [0x1000000504d8] +[b19n03:123615] [ 1] [0x3ffffbdeb040] +[b19n03:123615] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n03:123615] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n13:122032] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n13:122032] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n13:122032] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n13:122032] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n13:122032] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n14:168374] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n14:168374] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n14:168374] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n14:168374] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n14:168374] [ 7] [a27n11:23061] Signal: Segmentation fault (11) +[a27n11:23061] Signal code: (3) +[a27n11:23061] Failing at address: 0xb170000000000a0 +[a27n11:23061] [ 0] [0x1000000504d8] +[a27n11:23061] [ 1] [0x3fffcb3f5fc0] +[a27n11:23061] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n11:23061] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n08:172900] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n08:172900] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n08:172900] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n08:172900] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n08:172900] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n08:172900] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n08:172900] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n09:149099] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n09:149099] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n09:149099] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n09:149099] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n11:170897] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n11:170897] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n11:170897] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n01:128829] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n01:128829] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n01:128829] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n07:180194] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n07:180194] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n07:180194] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n07:180194] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n07:180194] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n07:180194] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n01:166961] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n01:166961] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n01:166961] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n01:166961] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n01:166961] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n01:166961] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n01:166961] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n01:166961] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n01:166961] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n01:166961] *** End of error message *** +[0x1000000504d8] +[b19n11:133970] [ 1] [0x3fffd4bf6240] +[b19n11:133970] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n11:133970] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n11:133970] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n11:133970] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n11:133970] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n11:133970] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n11:133970] [ 8] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n07:117359] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n03:38504] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n09:51331] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n09:51331] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n09:51331] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n09:51331] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n09:51331] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n16:135792] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n16:135792] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n16:135792] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n16:135792] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n16:135792] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n16:135792] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n16:135792] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n16:135792] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n16:135792] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n16:135792] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n01:48805] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n01:48805] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n01:48805] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n01:48805] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n01:48805] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n05:135165] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n05:135165] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n05:135165] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n05:135165] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n11:160348] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n11:160348] [ 5] [b08n12:12765] *** Process received signal *** +[b08n12:12765] Signal: Segmentation fault (11) +[b08n12:12765] Signal code: (3) +[b08n12:12765] Failing at address: 0xb170000000000a0 +[b08n12:12765] [ 0] [0x1000000504d8] +[b08n12:12765] [ 1] [0x3fffed749820] +[b08n12:12765] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n12:12765] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n15:177058] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n15:177058] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n15:177058] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n15:177058] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n12:167678] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n06:126374] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n06:126374] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n06:126374] [14] [b01n17:179115] [ 0] [0x1000000504d8] +[b01n17:179115] [ 1] [0x3ffff5cbfdc0] +[b01n17:179115] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n17:179115] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n17:179115] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n10:142929] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n10:142929] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n17:149904] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n17:149904] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n15:155432] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n15:155432] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n15:155432] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n15:155432] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n15:155432] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n15:155432] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n15:155432] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n16:116526] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n16:116526] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n16:116526] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n16:116526] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n13:114398] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n13:114398] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n13:114398] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n13:114398] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n13:114398] [10] [a31n05:158572] *** Process received signal *** +[a31n05:158572] Signal: Segmentation fault (11) +[a31n05:158572] Signal code: (3) +[a31n05:158572] Failing at address: 0xb170000000000a0 +[a31n05:158572] [ 0] [0x1000000504d8] +[a31n05:158572] [ 1] [0x3fffc7f50180] +[a31n05:158572] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n05:158572] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n05:158572] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n05:158572] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n05:158572] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n05:158572] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n05:158572] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n05:158572] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n05:158572] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n05:158572] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n05:158572] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n05:158572] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n05:158572] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n05:158572] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n16:157889] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n16:157889] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n16:157889] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n16:157889] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n08:118582] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n08:118582] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n08:118582] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n08:118582] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n08:118582] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n08:118582] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n08:118582] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n08:118582] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n08:118582] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n08:118582] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n08:118582] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n08:118582] *** End of error message *** +[0x3fffc8fdbba0] +[b19n09:123419] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n09:123419] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n09:123419] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n07:118167] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n07:118167] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n07:118167] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n02:06561] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n02:06561] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n14:128871] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n14:128871] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n14:128871] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n18:179699] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n18:179699] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n18:179699] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n18:179699] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n18:179699] [14] [b02n17:03843] *** Process received signal *** +[b02n17:03843] Signal: Segmentation fault (11) +[b02n17:03843] Signal code: (3) +[b02n17:03843] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n13:122144] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n13:122144] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n13:122144] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n13:122144] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n13:122144] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n13:122144] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n13:122144] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n13:122144] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n13:122144] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n13:122144] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n13:122144] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n14:133632] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n14:133632] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n14:133632] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n14:133632] [14] [0x3fffe09d66a0] +[b02n01:167298] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n01:167298] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n01:167298] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n01:167298] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n01:167298] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n13:176367] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n13:176367] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n13:176367] [ 6] [0x1000000504d8] +[a28n09:03927] [ 1] [0x3fffd407b300] +[a28n09:03927] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n09:03927] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n09:03927] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n09:03927] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n09:03927] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n09:03927] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n09:03927] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n12:122964] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n12:122964] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n14:169522] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n14:169522] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n14:169522] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n14:169522] [14] [0x1000000504d8] +[a28n10:176309] [ 1] [0x3fffc6164670] +[a28n10:176309] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n10:176309] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n10:176309] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n17:15873] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n17:15873] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n17:15873] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n17:15873] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n17:15873] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n01:141819] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n01:141819] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n01:141819] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n01:141819] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n06:01841] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n06:01841] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n06:01841] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n17:138142] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n17:138142] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n07:138201] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n07:138201] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n07:138201] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n03:177927] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n03:177927] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n03:177927] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n03:177927] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n03:177927] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n14:175714] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n14:175714] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n14:175714] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n14:175714] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n14:175714] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n14:175714] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n06:143011] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n14:159239] *** End of error message *** +[b05n08:08955] *** Process received signal *** +[b05n08:08955] Signal: Segmentation fault (11) +[b05n08:08955] Signal code: (3) +[b05n08:08955] Failing at address: 0x5170000000000a0 +[b05n08:08955] [ 0] [0x1000000504d8] +[b05n08:08955] [ 1] [0x3fffc9711970] +[b05n08:08955] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n08:08955] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n16:124664] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n16:124664] *** End of error message *** +[b02n02:166747] [ 0] [0x1000000504d8] +[b02n02:166747] [ 1] [0x3fffec5e6460] +[b02n02:166747] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n02:166747] [ 3] [0x1000000504d8] +[a35n14:136662] [ 1] [0x3fffd88be5d0] +[a35n14:136662] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n14:136662] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n14:136662] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n14:136662] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n14:136662] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n14:136662] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n14:136662] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n14:136662] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n14:136662] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n06:43019] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n06:43019] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n06:43019] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n06:43019] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n08:147822] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n08:147822] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n08:147822] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n08:147822] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n08:147822] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n18:126903] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n18:126903] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n07:132090] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n07:132090] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n07:132090] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n07:132090] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n10:67858] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n10:67858] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n10:67858] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n02:164489] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n02:164489] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n03:139011] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n03:139011] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n03:139011] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n03:139011] [13] [0x3fffc816ebf0] +[b09n11:171965] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n11:171965] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n11:171965] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n07:131671] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n07:131671] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n07:131671] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n07:131671] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n07:131671] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n07:131671] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n07:153178] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n07:153178] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n07:153178] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n07:153178] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n16:01571] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n16:01571] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n16:01571] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n16:01571] [13] [a29n02:163029] *** Process received signal *** +[a29n02:163029] Signal: Segmentation fault (11) +[a29n02:163029] Signal code: (3) +[a29n02:163029] Failing at address: 0x5170000000000a0 +[a29n02:163029] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n13:123708] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n13:123708] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n13:123708] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n13:123708] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n01:128157] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n03:10291] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n03:10291] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n03:10291] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n03:10291] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n03:10291] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n03:10291] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n13:156920] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n11:152988] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n11:152988] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n11:152988] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n11:152988] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n11:152988] *** End of error message *** +[ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n13:01298] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n13:01298] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n13:01298] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n05:138197] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n05:138197] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n05:138197] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n05:138197] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n05:138197] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n05:138197] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n05:138197] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n05:138197] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n05:138197] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n05:138197] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n04:169913] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n04:169913] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n04:169913] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n17:164294] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n17:164294] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n17:164294] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n17:164294] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n17:164294] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n17:164294] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n17:164294] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n17:164294] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n17:164294] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n17:164294] *** End of error message *** +[0x1000000504d8] +[a27n18:28216] [ 1] [0x3ffff0261470] +[a27n18:28216] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n18:28216] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n18:28216] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n16:153293] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n16:153293] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n16:153293] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n16:153293] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n17:150927] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n17:150927] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n08:174412] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n14:161617] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n14:161617] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n14:161617] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n14:161617] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n14:161617] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n14:161617] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n11:152841] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n08:124726] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n08:124726] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n08:124726] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n08:124726] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n08:124726] *** End of error message *** +[a28n11:01172] *** Process received signal *** +[a28n11:01172] Signal: Segmentation fault (11) +[a28n11:01172] Signal code: (3) +[a28n11:01172] Failing at address: 0xb170000000000a0 +[a28n11:01172] [ 0] [0x1000000504d8] +[a28n11:01172] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n09:173749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n17:156182] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n17:156182] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n02:143967] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n02:143967] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n02:143967] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n02:143967] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n02:143967] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n02:143967] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n02:143967] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n02:143967] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n02:143967] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n02:143967] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n02:143967] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n02:143967] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n10:130583] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n10:130583] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n10:130583] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n10:130583] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n09:121754] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n09:121754] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n09:121754] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n09:121754] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n09:121754] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n06:122164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n06:122164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n06:122164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n06:122164] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n06:122164] *** End of error message *** +[b05n12:11145] *** Process received signal *** +[b05n12:11145] Signal: Segmentation fault (11) +[b05n12:11145] Signal code: (3) +[b05n12:11145] Failing at address: 0xb170000000000a0 +[b05n12:11145] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n15:85939] *** End of error message *** +[0x3ffffc201e60] +[b12n01:145733] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n01:145733] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n01:145733] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n01:145733] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n01:145733] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n01:130410] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n01:130410] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n01:130410] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n09:119549] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n09:119549] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n09:119549] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n09:119549] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n09:119549] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n09:119549] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n09:119549] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n09:119549] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n09:119549] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n09:123812] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n09:123812] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n09:123812] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n09:123812] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n09:123812] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n09:123812] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n09:123812] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n09:123812] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n09:123812] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n09:123812] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n09:123812] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n16:171919] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n16:171919] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n16:171919] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n16:171919] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n01:173779] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n01:173779] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n01:173779] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n01:173779] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n01:173779] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n01:173779] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n01:173779] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n15:177860] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n15:177860] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n15:177860] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n15:177860] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n16:122494] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n16:122494] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n12:123234] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n12:123234] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n12:123234] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n12:123234] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n12:123234] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n03:127630] *** End of error message *** +[a30n12:148381] *** Process received signal *** +[a30n12:148381] Signal: Segmentation fault (11) +[a30n12:148381] Signal code: (3) +[a30n12:148381] Failing at address: 0x14170000000000a0 +[a30n12:148381] [ 0] [0x1000000504d8] +[a30n12:148381] [ 1] [0x3fffd7fc8270] +[a30n12:148381] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n12:148381] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n17:119583] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n17:119583] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n17:119583] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n11:137874] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n11:137874] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n11:137874] [ 6] [0x3fffeb9e70d0] +[b12n04:138159] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n04:138159] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n04:138159] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n04:138159] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n04:138159] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n04:176000] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n04:176000] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n04:176000] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n04:176000] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n12:156917] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n12:156917] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n12:156917] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n02:120190] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n02:120190] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n02:120190] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n15:148580] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n15:148580] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n15:148580] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n15:148580] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n15:148580] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n02:163321] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n02:163321] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n02:163321] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n17:164559] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n17:164559] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n17:164559] [ 8] [b18n10:122208] *** Process received signal *** +[b18n10:122208] Signal: Segmentation fault (11) +[b18n10:122208] Signal code: (3) +[b18n10:122208] Failing at address: 0x14170000000000a0 +[b18n10:122208] [ 0] [0x1000000504d8] +[b18n10:122208] [ 1] [0x3fffcc6d4510] +[b18n10:122208] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n10:122208] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n10:151893] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n10:151893] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n10:151893] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n10:151893] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n10:151893] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n10:151893] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n10:151893] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n10:151893] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n10:151893] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n10:151893] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n10:151893] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n10:151893] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n12:52454] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n12:52454] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n12:52454] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n12:52454] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n18:163106] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n18:163106] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n18:163106] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n18:163106] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n18:163106] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n18:163106] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n18:163573] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n18:163573] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n18:163573] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n18:163573] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n18:163573] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n18:163573] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n18:163573] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n18:163573] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n18:163573] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n18:163573] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n18:163573] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n18:163573] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n14:173895] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n14:173895] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n14:173895] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n14:173895] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n09:122068] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n09:122068] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n09:122068] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n09:122068] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n09:122068] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n08:129888] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n08:129888] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n08:129888] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n08:129888] [10] [0x3fffea92fc90] +[b09n06:167090] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n06:167090] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n06:167090] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n06:167090] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n06:167090] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n06:167090] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n06:167090] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n06:167090] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n06:167090] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n06:167090] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n06:167090] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n06:167090] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n06:167090] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n06:167090] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n04:164047] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n04:164047] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n06:155088] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n06:155088] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n06:155088] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n06:155088] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n06:155088] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n06:155088] *** End of error message *** +[b17n07:125473] *** Process received signal *** +[b17n07:125473] Signal: Segmentation fault (11) +[b17n07:125473] Signal code: (3) +[b17n07:125473] Failing at address: 0x5170000000000a0 +[b17n07:125473] [ 0] [0x1000000504d8] +[b17n07:125473] [ 1] [0x3ffff1725920] +[b17n07:125473] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n07:125473] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n03:168210] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n03:168210] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n08:163160] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n08:163160] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n08:163160] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n08:163160] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n08:163160] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n11:176299] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n07:127354] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n07:127354] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n07:127354] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n09:152061] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n09:152061] [ 5] [a30n08:163099] *** Process received signal *** +[a30n08:163099] Signal: Segmentation fault (11) +[a30n08:163099] Signal code: (3) +[a30n08:163099] Failing at address: 0xb170000000000a0 +[a30n08:163099] [ 0] [0x1000000504d8] +[a30n08:163099] [ 1] [0x3fffcecee110] +[a30n08:163099] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n08:163099] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n01:165165] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n01:165165] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n01:165165] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n01:165165] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n01:165165] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n01:165165] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n01:165165] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n15:154386] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n15:154386] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n15:154386] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n15:154386] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n15:154386] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n15:154386] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n15:154386] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n15:154386] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n15:154386] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n15:154386] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n15:154386] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n15:154386] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n15:154386] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n08:156371] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n08:156371] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n08:156371] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n08:156371] [ 9] [b07n13:165261] *** Process received signal *** +[b07n13:165261] Signal: Segmentation fault (11) +[b07n13:165261] Signal code: (3) +[b07n13:165261] Failing at address: 0x14170000000000a0 +[b07n13:165261] [ 0] [0x1000000504d8] +[b07n13:165261] [ 1] [0x3fffff34a190] +[b07n13:165261] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n13:163194] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n13:163194] [ 5] [a27n06:60154] *** Process received signal *** +[a27n06:60154] Signal: Segmentation fault (11) +[a27n06:60154] Signal code: (3) +[a27n06:60154] Failing at address: 0x14170000000000a0 +[a27n06:60154] [ 0] [0x1000000504d8] +[a27n06:60154] [ 1] [0x3fffd6436ce0] +[a27n06:60154] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n06:60154] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n06:60154] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n06:60154] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n06:60154] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n06:60154] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n06:60154] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n06:60154] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n06:60154] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n06:60154] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n06:60154] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n06:60154] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n06:60154] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n06:60154] *** End of error message *** +[b07n15:166701] *** Process received signal *** +[b07n15:166701] Signal: Segmentation fault (11) +[b07n15:166701] Signal code: (3) +[b07n15:166701] Failing at address: 0xb170000000000a0 +[b07n15:166701] [ 0] [0x1000000504d8] +[b07n15:166701] [ 1] [0x3fffcd51ba90] +[b07n15:166701] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n15:166701] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n15:166701] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n15:166701] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n15:166701] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n15:166701] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n15:166701] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n15:166701] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n15:166701] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n15:166701] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n15:166701] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n15:166701] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n15:166701] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n15:166701] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n16:166689] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n16:166689] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n16:166689] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n16:166689] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n16:166689] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n16:166689] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n16:166689] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n16:166689] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n16:166689] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n16:166689] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n18:145743] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n18:145743] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n18:145743] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n18:145743] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n18:145743] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n02:155379] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n02:155379] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n02:155379] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n02:155379] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n02:155379] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n02:155379] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n02:155379] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n02:155379] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n02:155379] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n11:147712] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n11:147712] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n11:147712] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n12:115636] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n12:115636] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n12:115636] [11] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n06:38118] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n06:38118] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n06:38118] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n06:38118] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n06:38118] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n06:38118] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n06:38118] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n06:38118] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n15:03734] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n15:03734] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n15:03734] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n15:03734] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n15:03734] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n15:03734] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n15:03734] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n15:03734] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n15:03734] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n15:03734] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n15:03734] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n15:03734] *** End of error message *** +[b10n04:169410] *** Process received signal *** +[b10n04:169410] Signal: Segmentation fault (11) +[b10n04:169410] Signal code: (3) +[b10n04:169410] Failing at address: 0xb170000000000a0 +[b10n04:169410] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n05:148377] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n05:148377] [ 6] [b05n06:177846] *** Process received signal *** +[b05n06:177846] Signal: Segmentation fault (11) +[b05n06:177846] Signal code: (3) +[b05n06:177846] Failing at address: 0xb170000000000a0 +[b05n06:177846] [ 0] [0x1000000504d8] +[b05n06:177846] [ 1] [0x3fffd7d43be0] +[b05n06:177846] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n06:177846] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n06:177846] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n06:177846] [ 5] [b04n05:169785] *** Process received signal *** +[b06n12:177120] *** Process received signal *** +[b06n12:177120] Signal: Segmentation fault (11) +[b06n12:177120] Signal code: (3) +[b06n12:177120] Failing at address: 0x14170000000000a0 +[b06n12:177120] [ 0] [0x1000000504d8] +[b06n12:177120] [ 1] [0x3fffdb70a330] +[b06n12:177120] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n12:177120] [ 3] [b03n11:175595] [ 0] [0x1000000504d8] +[b03n11:175595] [ 1] [0x3fffe9b49e00] +[b03n11:175595] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n11:175595] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n11:175595] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n09:79006] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n09:79006] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n09:79006] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n08:121687] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n08:121687] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n08:121687] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n08:121687] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n08:121687] [13] [b02n03:39687] *** Process received signal *** +[b02n03:39687] Signal: Segmentation fault (11) +[b02n03:39687] Signal code: (3) +[b02n03:39687] Failing at address: 0x14170000000000a0 +[b02n03:39687] [ 0] [0x1000000504d8] +[b02n03:39687] [ 1] [0x3fffcb3b1180] +[b02n03:39687] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n03:39687] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n03:39687] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n03:39687] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n03:39687] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n03:39687] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n03:39687] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n03:39687] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n03:39687] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n03:39687] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n03:39687] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n03:39687] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n03:39687] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n03:39687] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n12:154342] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n12:154342] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n12:154342] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n12:154342] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n11:120241] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n11:120241] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n11:120241] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n11:120241] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n15:137007] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n15:137007] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n02:149227] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n02:149227] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n02:149227] [ 5] [b04n06:10225] *** Process received signal *** +[b04n06:10225] Signal: Segmentation fault (11) +[b04n06:10225] Signal code: (3) +[b04n06:10225] Failing at address: 0x5170000000000a0 +[b04n06:10225] [ 0] [0x1000000504d8] +[b04n06:10225] [ 1] [0x3fffd43df570] +[a35n01:138004] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n01:138004] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n01:138004] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n01:138004] [ 5] [0x1000000504d8] +[a27n02:49105] [ 1] [0x3ffff2990d00] +[a27n02:49105] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n02:49105] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n02:49105] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n11:152813] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n11:152813] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n11:152813] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n11:152813] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n11:152813] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n11:152813] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n11:152813] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n11:152813] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n11:152813] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n11:152813] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n15:123041] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n15:123041] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n15:123041] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n11:02684] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n11:02684] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n11:02684] [ 8] [b13n12:140121] *** Process received signal *** +[b13n12:140121] Signal: Segmentation fault (11) +[b13n12:140121] Signal code: (3) +[b13n12:140121] Failing at address: 0xb170000000000a0 +[b13n12:140121] [ 0] [0x1000000504d8] +[b13n12:140121] [ 1] [a35n17:146794] *** Process received signal *** +[a35n17:146794] Signal: Segmentation fault (11) +[a35n17:146794] Signal code: (3) +[a35n17:146794] Failing at address: 0xb170000000000a0 +[a35n17:146794] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n09:164364] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n03:168929] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n03:168929] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n03:168929] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n03:168929] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n03:168929] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n03:168929] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n03:168929] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n03:135164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n03:135164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n03:135164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n03:135164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n03:135164] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n03:135164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n03:135164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n03:135164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n03:135164] [14] [b04n12:11418] *** Process received signal *** +[b04n12:11418] Signal: Segmentation fault (11) +[b04n12:11418] Signal code: (3) +[b04n12:11418] Failing at address: 0x14170000000000a0 +[b04n12:11418] [ 0] [0x1000000504d8] +[b04n12:11418] [ 1] [0x3ffff42b6c50] +[b04n12:11418] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n12:11418] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n17:119467] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n17:119467] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n17:119467] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n17:126526] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n17:126526] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n17:126526] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n17:126526] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n14:163550] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n14:163550] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n14:163550] [ 6] [0x3fffdfab2c90] +[b06n15:161757] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n15:161757] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n15:161757] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n01:153683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n01:153683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n01:153683] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n14:21317] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n14:21317] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n14:21317] [ 6] [b22n16:124793] *** Process received signal *** +[b22n16:124793] Signal: Segmentation fault (11) +[b22n16:124793] Signal code: (3) +[b22n16:124793] Failing at address: 0xb170000000000a0 +[b22n16:124793] [ 0] [0x3fffeedd3c10] +[b07n06:177004] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n06:177004] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n06:177004] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n06:177004] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n09:147790] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n09:147790] [ 6] [a30n03:167775] *** Process received signal *** +[a30n03:167775] Signal: Segmentation fault (11) +[a30n03:167775] Signal code: (3) +[a30n03:167775] Failing at address: 0x5170000000000a0 +[a30n03:167775] [ 0] [0x1000000504d8] +[a30n03:167775] [ 1] [0x3fffcdbc25d0] +[a30n03:167775] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n03:167775] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n03:167775] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n03:167775] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n03:167775] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n03:167775] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n03:167775] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n03:167775] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n03:167775] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n03:167775] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n03:167775] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n03:167775] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n03:167775] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n03:167775] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n02:178473] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n02:178473] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n02:178473] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n02:178473] [ 6] [a26n01:65728] *** Process received signal *** +[a26n01:65728] Signal: Segmentation fault (11) +[a26n01:65728] Signal code: (3) +[a26n01:65728] Failing at address: 0x14170000000000a0 +[a26n01:65728] [ 0] [0x1000000504d8] +[a26n01:65728] [ 1] [0x3fffc5aedca0] +[a26n01:65728] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n01:65728] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n03:125240] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n03:125240] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n03:125240] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n03:125240] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n03:125240] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n03:125240] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n03:125240] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n05:131261] [ 5] [b11n05:162565] *** End of error message *** +[b06n05:163962] *** Process received signal *** +[b06n05:163962] Signal: Segmentation fault (11) +[b06n05:163962] Signal code: (3) +[b06n05:163962] Failing at address: 0x14170000000000a0 +[b06n05:163962] [ 0] [0x1000000504d8] +[b06n05:163962] [ 1] [0x3ffff85408e0] +[b06n05:163962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n05:163962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n05:163962] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n05:163962] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n05:163962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n05:163962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n05:163962] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n05:163962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n05:163962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n05:163962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n05:163962] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n05:163962] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n05:163962] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n05:163962] *** End of error message *** +[b01n04:169704] *** Process received signal *** +[b01n04:169704] Signal: Segmentation fault (11) +[b01n04:169704] Signal code: (3) +[b01n04:169704] Failing at address: 0xb170000000000a0 +[b01n04:169704] [ 0] [0x1000000504d8] +[b01n04:169704] [ 1] [0x3fffe2e7e100] +[b01n04:169704] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n04:169704] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n04:169704] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n03:01352] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n03:01352] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n03:01352] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n03:01352] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n04:118344] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n04:118344] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n01:164963] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n01:164963] [11] [0x1000000504d8] +[a26n04:54378] [ 1] [0x3fffc9d8bec0] +[a26n04:54378] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n04:54378] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n04:54378] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n04:54378] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n04:54378] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n04:54378] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n04:54378] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n04:54378] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n04:54378] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n04:54378] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n04:54378] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n04:54378] [13] [b17n02:127218] *** Process received signal *** +[b17n02:127218] Signal: Segmentation fault (11) +[b17n02:127218] Signal code: (3) +[b17n02:127218] Failing at address: 0x14170000000000a0 +[b17n02:127218] [ 0] [0x1000000504d8] +[b17n02:127218] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n03:136668] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n03:136668] [ 5] [a32n06:154623] *** Process received signal *** +[a32n06:154623] Signal: Segmentation fault (11) +[a32n06:154623] Signal code: (3) +[a32n06:154623] Failing at address: 0x5170000000000a0 +[a32n06:154623] [ 0] [0x1000000504d8] +[a32n06:154623] [ 1] [b19n02:123704] *** Process received signal *** +[b19n02:123704] Signal: Segmentation fault (11) +[b19n02:123704] Signal code: (3) +[b19n02:123704] Failing at address: 0xb170000000000a0 +[b19n02:123704] [ 0] [0x1000000504d8] +[b19n02:123704] [ 1] [0x3ffff63346b0] +[b19n02:123704] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n02:123704] [ 3] [b11n10:161716] *** Process received signal *** +[b11n10:161716] Signal: Segmentation fault (11) +[b11n10:161716] Signal code: (3) +[b11n10:161716] Failing at address: 0x14170000000000a0 +[b11n10:161716] [ 0] [0x1000000504d8] +[b11n10:161716] [ 1] [0x3fffd91d78f0] +[b11n10:161716] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n10:161716] [ 3] [0x3ffff05260b0] +[b06n17:163816] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n17:163816] [ 3] [0x3fffc95443c0] +[a28n01:179263] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n01:179263] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n01:179263] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n01:179263] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n01:179263] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n08:172427] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n08:172427] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n08:172427] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n08:172427] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n07:03459] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n07:03459] [11] [b07n17:162358] *** Process received signal *** +[b07n17:162358] Signal: Segmentation fault (11) +[b07n17:162358] Signal code: (3) +[b07n17:162358] Failing at address: 0xb170000000000a0 +[b07n17:162358] [ 0] [0x3fffc756d510] +[a35n11:148675] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n11:148675] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n11:148675] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n11:148675] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n06:130543] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n06:130543] [10] [0x1000000504d8] +[b18n06:133953] [ 1] [0x3fffdc571560] +[b18n06:133953] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n06:133953] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n06:133953] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n06:133953] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n09:174689] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n09:174689] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n09:174689] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n09:174689] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n09:174689] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n09:174689] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n09:174689] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n04:13081] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n04:13081] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n04:13081] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n04:13081] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n04:13081] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n05:135123] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n05:135123] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n05:135123] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n05:135123] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n05:135123] [ 8] [a32n03:151880] *** Process received signal *** +[a32n03:151880] Signal: Segmentation fault (11) +[a32n03:151880] Signal code: (3) +[a32n03:151880] Failing at address: 0xb170000000000a0 +[a32n03:151880] [ 0] [0x1000000504d8] +[a32n03:151880] [ 1] [a28n08:175805] *** Process received signal *** +[a28n08:175805] Signal: Segmentation fault (11) +[a28n08:175805] Signal code: (3) +[a28n08:175805] Failing at address: 0x5170000000000a0 +[a28n08:175805] [ 0] [0x1000000504d8] +[a28n08:175805] [ 1] [0x3ffffce7b4f0] +[a28n08:175805] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n08:175805] [ 3] [0x3fffef6cd820] +[b15n02:122556] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n02:122556] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n02:122556] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n02:147885] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n02:147885] [ 5] [b07n09:163513] *** Process received signal *** +[b07n09:163513] Signal: Segmentation fault (11) +[b07n09:163513] Signal code: (3) +[b07n09:163513] Failing at address: 0xb170000000000a0 +[b07n09:163513] [ 0] [0x1000000504d8] +[b07n09:163513] [ 1] [0x3ffff0e16ea0] +[b07n09:163513] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n09:163513] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n09:163513] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n09:163513] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n09:163513] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n09:163513] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n09:163513] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n09:163513] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n09:163513] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n09:163513] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n09:163513] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n09:163513] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n09:163513] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n05:54535] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n05:54535] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n05:54535] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n05:54535] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n04:142256] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n04:142256] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n04:142256] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n04:142256] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n11:164996] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n11:164996] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n11:164996] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n11:164996] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n11:164996] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n11:164996] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n11:164996] *** End of error message *** +[a36n03:153849] *** Process received signal *** +[b21n05:130803] *** Process received signal *** +[b21n05:130803] Signal: Segmentation fault (11) +[b21n05:130803] Signal code: (3) +[b21n05:130803] Failing at address: 0xb170000000000a0 +[b21n05:130803] [ 0] [0x1000000504d8] +[b21n05:130803] [ 1] [b08n01:171785] *** Process received signal *** +[b08n01:171785] Signal: Segmentation fault (11) +[b08n01:171785] Signal code: (3) +[b08n01:171785] Failing at address: 0x14170000000000a0 +[b08n01:171785] [ 0] [0x1000000504d8] +[b08n01:171785] [ 1] [0x1000000504d8] +[b21n17:118575] [ 1] [0x3fffde9309b0] +[b21n17:118575] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n17:118575] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n17:118575] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n17:118575] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n18:118680] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n18:118680] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n10:164940] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n10:164940] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n10:164940] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n10:164940] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n10:164940] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n01:141341] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n01:141341] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n01:141341] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n05:154186] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n05:154186] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n05:154186] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n05:154186] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n05:154186] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n05:154186] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n05:154186] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n05:154186] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n05:154186] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n05:154186] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n05:154186] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n05:154186] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n02:121683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n02:121683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n02:121683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n02:121683] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n16:121928] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n04:04849] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n04:04849] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n04:04849] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n04:04849] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n04:04849] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n04:04849] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n04:04849] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n06:162919] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n01:158305] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n01:158305] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n01:158305] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n01:158305] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n01:158305] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n01:158305] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n01:158305] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n01:158305] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n01:158305] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n16:171167] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n16:171167] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n16:171167] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n16:171167] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n16:171167] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n04:123055] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n04:123055] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n04:123055] [ 6] [a36n09:140253] *** Process received signal *** +[a36n09:140253] Signal: Segmentation fault (11) +[a36n09:140253] Signal code: (3) +[a36n09:140253] Failing at address: 0xb170000000000a0 +[a36n09:140253] [ 0] [0x1000000504d8] +[a36n09:140253] [ 1] [0x3ffff905b460] +[a36n09:140253] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n09:140253] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n09:140253] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n09:140253] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n09:140253] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n09:140253] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n09:140253] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n09:140253] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n09:140253] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n09:140253] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n09:140253] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n09:140253] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n09:140253] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n09:140253] *** End of error message *** +[b08n08:164752] *** Process received signal *** +[b08n08:164752] Signal: Segmentation fault (11) +[b08n08:164752] Signal code: (3) +[b08n08:164752] Failing at address: 0x14170000000000a0 +[b08n08:164752] [ 0] [0x1000000504d8] +[b08n08:164752] [ 1] [0x3fffc85cd080] +[b08n08:164752] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n08:164752] [ 3] [b01n01:146655] [ 0] [0x1000000504d8] +[b01n01:146655] [ 1] [0x3fffc24c7020] +[b01n01:146655] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n01:146655] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n01:146655] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n01:146655] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n01:146655] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n01:146655] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n01:146655] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n01:146655] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n01:146655] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n01:146655] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n01:146655] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n01:146655] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n01:146655] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n01:146655] *** End of error message *** +[b22n06:133054] *** Process received signal *** +[b22n06:133054] Signal: Segmentation fault (11) +[b22n06:133054] Signal code: (3) +[b22n06:133054] Failing at address: 0xb170000000000a0 +[b22n06:133054] [ 0] [0x1000000504d8] +[b22n06:133054] [ 1] [0x3fffd89f0010] +[b22n06:133054] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n06:133054] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n06:133054] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n06:133054] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n06:133054] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n06:133054] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n06:133054] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n06:133054] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n06:133054] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n06:133054] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n06:133054] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n06:133054] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n06:133054] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n06:133054] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n01:132063] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n01:132063] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n01:132063] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n03:121809] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n03:121809] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n03:121809] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n03:121809] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n03:121809] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n03:121809] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n03:121809] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n03:121809] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n03:121809] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n03:121809] *** End of error message *** +[b02n15:169490] *** Process received signal *** +[b02n15:169490] Signal: Segmentation fault (11) +[b02n15:169490] Signal code: (3) +[b02n15:169490] Failing at address: 0xb170000000000a0 +[b02n15:169490] [ 0] [0x1000000504d8] +[b02n15:169490] [ 1] [0x3fffebdda9f0] +[b02n15:169490] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n15:169490] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n15:169490] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n15:169490] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n15:169490] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n15:169490] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n15:169490] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n15:169490] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n15:169490] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n15:169490] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n15:169490] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n15:169490] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n15:169490] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n15:169490] *** End of error message *** +[a36n14:152036] *** Process received signal *** +[a36n14:152036] Signal: Segmentation fault (11) +[a36n14:152036] Signal code: (3) +[a36n14:152036] Failing at address: 0xb170000000000a0 +[a36n14:152036] [ 0] [0x1000000504d8] +[a36n14:152036] [ 1] [b15n04:133901] *** Process received signal *** +[b15n04:133901] Signal: Segmentation fault (11) +[b15n04:133901] Signal code: (3) +[b15n04:133901] Failing at address: 0x14170000000000a0 +[b15n04:133901] [ 0] [0x1000000504d8] +[b15n04:133901] [ 1] [0x3ffff6aa8430] +[b15n04:133901] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n04:133901] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n04:133901] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n04:133901] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n04:133901] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n04:133901] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n04:133901] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n04:133901] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n04:133901] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n04:133901] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n04:133901] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n04:133901] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n04:133901] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n04:133901] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n06:171504] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n06:171504] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n06:171504] [ 6] [0x3fffd8d5ff50] +[b16n06:132107] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n06:132107] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n06:132107] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n06:132107] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n06:132107] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n06:132107] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n06:132107] [ 8] [b23n02:116887] *** Process received signal *** +[b23n02:116887] Signal: Segmentation fault (11) +[b23n02:116887] Signal code: (3) +[b23n02:116887] Failing at address: 0x14170000000000a0 +[b23n02:116887] [ 0] [0x1000000504d8] +[b23n02:116887] [ 1] [0x3fffe568b160] +[b23n02:116887] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n02:116887] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n02:116887] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n05:02908] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n05:02908] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n05:02908] [ 6] [b15n06:120733] *** Process received signal *** +[b15n06:120733] Signal: Segmentation fault (11) +[b15n06:120733] Signal code: (3) +[b15n06:120733] Failing at address: 0xb170000000000a0 +[b15n06:120733] [ 0] [0x1000000504d8] +[b15n06:120733] [ 1] [0x3ffff096dde0] +[b15n06:120733] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n06:120733] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n06:120733] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n06:120733] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n07:119636] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n07:119636] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n07:119636] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n07:119636] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n07:119636] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n07:119636] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n07:119636] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n07:119636] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n07:119636] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n07:119636] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n07:119636] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n07:119636] *** End of error message *** +[b10n18:162419] *** Process received signal *** +[b10n18:162419] Signal: Segmentation fault (11) +[b10n18:162419] Signal code: (3) +[b10n18:162419] Failing at address: 0x14170000000000a0 +[b10n18:162419] [ 0] [0x1000000504d8] +[b10n18:162419] [ 1] [0x3fffe47696d0] +[b10n18:162419] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n18:162419] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n18:162419] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n18:162419] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n18:162419] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n18:162419] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n18:162419] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n18:162419] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n18:162419] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n18:162419] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n18:162419] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n18:162419] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n18:162419] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n18:162419] *** End of error message *** +[b03n07:03372] [ 0] [0x1000000504d8] +[b03n07:03372] [ 1] [0x3fffde625750] +[b03n07:03372] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n07:03372] [ 3] [b03n10:02711] Signal: Segmentation fault (11) +[b03n10:02711] Signal code: (3) +[b03n10:02711] Failing at address: 0x14170000000000a0 +[b03n10:02711] [ 0] [0x1000000504d8] +[b03n10:02711] [ 1] [0x3fffeef13030] +[b03n10:02711] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n10:02711] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n10:02711] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n10:02711] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n10:02711] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n08:121434] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n08:121434] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n08:121434] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n08:121434] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n08:121434] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n08:121434] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n08:121434] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n08:121434] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n08:121434] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n08:121434] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n05:122271] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n05:122271] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n05:122271] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n05:122271] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n05:122271] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n05:122271] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n05:122271] [10] [0x3fffdc9e8010] +[b14n03:130345] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n03:130345] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n03:130345] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n03:130345] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n03:130345] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n03:130345] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n03:130345] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n03:130345] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n03:130345] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n02:139023] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n02:139023] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n02:139023] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n06:163677] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n06:163677] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n06:163677] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n11:160798] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n11:160798] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n11:160798] [ 6] [b10n05:173080] [ 0] [0x1000000504d8] +[b10n05:173080] [ 1] [0x3fffc4905850] +[b10n05:173080] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n05:173080] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n01:128004] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n01:128004] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n05:120333] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n05:120333] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n05:120333] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n05:120333] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n05:120333] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n12:138737] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n12:138737] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n16:164549] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n16:164549] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n04:126860] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n04:126860] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n04:126860] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n04:126860] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n04:126860] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n04:126860] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n04:126860] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n04:126860] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n04:126860] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n04:126860] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n04:126860] *** End of error message *** +[a32n15:139058] *** Process received signal *** +[a32n15:139058] Signal: Segmentation fault (11) +[a32n15:139058] Signal code: (3) +[a32n15:139058] Failing at address: 0x14170000000000a0 +[a32n15:139058] [ 0] [0x1000000504d8] +[a32n15:139058] [ 1] [0x3fffdfc13ff0] +[a32n15:139058] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n15:139058] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n15:139058] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n15:139058] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n15:139058] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n15:139058] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n15:139058] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n15:139058] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n15:139058] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n15:139058] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n15:139058] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n15:139058] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n15:139058] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n15:139058] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n05:126781] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n05:126781] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n05:126781] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n05:126781] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n05:126781] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n05:126781] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n05:126781] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n07:124352] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n07:124352] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n07:124352] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n09:155756] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n09:155756] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n09:155756] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n09:155756] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n09:155756] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n17:176413] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n17:176413] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n17:176413] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n17:123235] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n17:123235] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n17:123235] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n17:123235] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n17:123235] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n17:123235] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n17:123235] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n17:123235] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n17:123235] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n17:123235] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n13:10198] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n13:10198] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n13:10198] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n13:10198] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n13:10198] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n13:10198] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n13:10198] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n13:10198] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n13:10198] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n13:10198] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n10:16972] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n10:16972] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n10:16972] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n10:16972] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n10:16972] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n10:16972] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n10:16972] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n10:16972] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n10:16972] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n01:165125] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n01:165125] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n01:165125] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n01:165125] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n01:165125] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n01:165125] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n01:165125] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n01:165125] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n01:165125] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n11:117476] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n11:117476] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n11:117476] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n11:117476] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n11:117476] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n06:138765] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n06:138765] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n06:138765] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n06:138765] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n06:161721] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n06:161721] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n06:161721] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n06:161721] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n06:161721] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n12:178597] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n17:168151] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n17:168151] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n17:168151] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n17:168151] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n06:117525] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n07:152083] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n07:152083] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n07:152083] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n07:152083] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n07:152083] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n07:152083] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n07:152083] [10] [a29n06:157789] [ 0] [0x1000000504d8] +[a29n06:157789] [ 1] [0x3fffe3df3f40] +[a29n06:157789] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n06:157789] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n06:157789] [ 4] [b11n13:153676] *** Process received signal *** +[b11n13:153676] Signal: Segmentation fault (11) +[b11n13:153676] Signal code: (3) +[b11n13:153676] Failing at address: 0xb170000000000a0 +[b11n13:153676] [ 0] [0x1000000504d8] +[b11n13:153676] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n11:03986] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n11:03986] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n11:03986] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n11:120581] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n11:120581] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n11:120581] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n11:120581] [10] [a27n03:46690] *** Process received signal *** +[a27n03:46690] Signal: Segmentation fault (11) +[a27n03:46690] Signal code: (3) +[a27n03:46690] Failing at address: 0x14170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n14:178327] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n14:178327] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n14:178327] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n07:160802] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n07:160802] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n07:160802] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n07:160802] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n07:160802] [ 9] [0x3fffc73050f0] +[b22n03:120668] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n03:120668] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n03:120668] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n03:120668] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n03:120668] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n04:158636] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n04:158636] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n04:158636] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n04:158636] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n03:154294] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n03:154294] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n03:154294] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n03:154294] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n03:154294] [ 9] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n04:145749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n04:145749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n05:139433] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n05:139433] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n05:139433] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n05:139433] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n05:139433] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n13:171110] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n13:171110] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n13:171110] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n13:171110] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n09:119634] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n09:119634] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n09:119634] [13] [b22n05:123936] *** Process received signal *** +[b22n05:123936] Signal: Segmentation fault (11) +[b22n05:123936] Signal code: (3) +[b22n05:123936] Failing at address: 0x14170000000000a0 +[b22n05:123936] [ 0] [0x1000000504d8] +[b22n05:123936] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n07:168964] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n07:168964] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n07:168964] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n07:168964] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n07:168964] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n07:168964] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n10:11488] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n10:11488] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n10:11488] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n10:11488] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n10:11488] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n12:141260] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n12:141260] *** End of error message *** +[a29n17:169734] *** Process received signal *** +[a29n17:169734] Signal: Segmentation fault (11) +[a29n17:169734] Signal code: (3) +[a29n17:169734] Failing at address: 0xb170000000000a0 +[a29n17:169734] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n17:119978] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n17:119978] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n17:119978] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n17:119978] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n17:119978] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n18:179171] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n18:179171] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n18:179171] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n18:179171] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n18:179171] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n16:08385] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n16:08385] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n16:08385] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n16:08385] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n16:08385] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n04:44755] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n04:44755] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n04:44755] [13] [b13n11:134519] *** Process received signal *** +[b13n11:134519] Signal: Segmentation fault (11) +[b13n11:134519] Signal code: (3) +[b13n11:134519] Failing at address: 0xb170000000000a0 +[b13n11:134519] [ 0] [0x1000000504d8] +[b13n11:134519] [ 1] [0x3fffd6b80af0] +[b13n11:134519] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n11:134519] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n05:139910] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n03:06824] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n03:06824] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n03:06824] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n03:06824] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n03:06824] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n13:123516] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n13:123516] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n13:123516] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n13:123516] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n12:123559] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n12:123559] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n12:123559] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n12:123559] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n12:123559] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n14:08398] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n14:08398] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n14:08398] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n14:08398] [14] [a30n05:156009] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n13:123054] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n13:123054] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n13:123054] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n13:123054] [14] [a30n13:154230] *** Process received signal *** +[a30n13:154230] Signal: Segmentation fault (11) +[a30n13:154230] Signal code: (3) +[a30n13:154230] Failing at address: 0xb170000000000a0 +[a30n13:154230] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n15:166119] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n15:166119] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n15:166119] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n15:166119] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n15:166119] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n05:174781] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n05:174781] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n05:174781] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n05:174781] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n05:174781] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n05:174781] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n05:174781] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n05:174781] [14] [a30n04:03147] *** Process received signal *** +[a30n04:03147] Signal: Segmentation fault (11) +[a30n04:03147] Signal code: (3) +[a30n04:03147] Failing at address: 0x14170000000000a0 +[a30n04:03147] [ 0] [0x1000000504d8] +[a30n04:03147] [ 1] [0x3fffd86341c0] +[a30n04:03147] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n04:03147] [ 3] [b15n14:133968] *** Process received signal *** +[b15n14:133968] Signal: Segmentation fault (11) +[b15n14:133968] Signal code: (3) +[b15n14:133968] Failing at address: 0xb170000000000a0 +[b15n14:133968] [ 0] [0x1000000504d8] +[b15n14:133968] [ 1] [0x3ffff35393f0] +[b15n14:133968] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n14:133968] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n14:133968] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n14:133968] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n14:133968] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n14:133968] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n14:133968] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n14:133968] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n14:133968] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n14:133968] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n14:133968] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n14:133968] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n14:133968] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n14:133968] *** End of error message *** +[a30n01:163390] Signal: Segmentation fault (11) +[a30n01:163390] Signal code: (3) +[a30n01:163390] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n16:53796] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n16:53796] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n16:53796] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n16:53796] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n08:141873] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n08:141873] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n08:141873] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n08:141873] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n08:176679] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n08:176679] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n08:176679] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n08:176679] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n08:176679] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n10:118387] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n10:118387] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n10:118387] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n10:118387] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n10:118387] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n12:160964] [10] [a31n16:162626] *** Process received signal *** +[a31n16:162626] Signal: Segmentation fault (11) +[a31n16:162626] Signal code: (3) +[a31n16:162626] Failing at address: 0xb170000000000a0 +[a31n16:162626] [ 0] [0x1000000504d8] +[a31n16:162626] [ 1] [0x3fffe48907e0] +[a31n16:162626] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n16:162626] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n16:162626] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n16:162626] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n16:162626] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n16:162626] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n16:162626] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n16:162626] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n16:162626] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n16:162626] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n16:162626] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n16:162626] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n16:162626] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n16:162626] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n10:169208] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n10:169208] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n10:169208] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n10:169208] [14] [b04n01:96754] *** Process received signal *** +[b04n01:96754] Signal: Segmentation fault (11) +[b04n01:96754] Signal code: (3) +[b04n01:96754] Failing at address: 0x5170000000000a0 +[b04n01:96754] [ 0] [0x1000000504d8] +[b04n01:96754] [ 1] [b14n11:130804] [ 0] [0x1000000504d8] +[b14n11:130804] [ 1] [0x3ffff791dbe0] +[b14n11:130804] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n11:130804] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n09:12247] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n09:12247] *** End of error message *** +[a34n12:145496] *** Process received signal *** +[a34n12:145496] Signal: Segmentation fault (11) +[a34n12:145496] Signal code: (3) +[a34n12:145496] Failing at address: 0x14170000000000a0 +[a34n12:145496] [ 0] [0x1000000504d8] +[a34n12:145496] [ 1] [0x3ffff8d30730] +[a34n12:145496] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n12:145496] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n15:120133] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n15:120133] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n15:120133] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n15:120133] [13] [a28n06:179450] *** Process received signal *** +[a28n06:179450] Signal: Segmentation fault (11) +[a28n06:179450] Signal code: (3) +[a28n06:179450] Failing at address: 0x14170000000000a0 +[a28n06:179450] [ 0] [0x1000000504d8] +[a28n06:179450] [ 1] [0x3fffd51849c0] +[a28n06:179450] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n06:179450] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n06:179450] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n06:179450] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n06:179450] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n06:179450] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n06:179450] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n06:179450] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n06:179450] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n06:179450] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n06:179450] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n06:179450] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n06:179450] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n06:179450] *** End of error message *** +[b03n01:170768] *** Process received signal *** +[b03n01:170768] Signal: Segmentation fault (11) +[b03n01:170768] Signal code: (3) +[b03n01:170768] Failing at address: 0x14170000000000a0 +[b03n01:170768] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n17:138615] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n17:138615] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n17:138615] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n17:138615] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n17:138615] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n17:138615] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n17:138615] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n17:138615] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n17:138615] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n17:138615] *** End of error message *** +[a29n08:159026] *** Process received signal *** +[a29n08:159026] Signal: Segmentation fault (11) +[a29n08:159026] Signal code: (3) +[a29n08:159026] Failing at address: 0x14170000000000a0 +[a29n08:159026] [ 0] [0x1000000504d8] +[a29n08:159026] [ 1] [0x3fffca5c8b40] +[a29n08:159026] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n08:159026] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n08:159026] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n08:159026] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n10:131275] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n10:131275] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n05:120231] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n05:120231] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n05:120231] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n05:120231] [14] [a33n10:140415] *** Process received signal *** +[a33n10:140415] Signal: Segmentation fault (11) +[a33n10:140415] Signal code: (3) +[a33n10:140415] Failing at address: 0x14170000000000a0 +[a33n10:140415] [ 0] [0x1000000504d8] +[a33n10:140415] [ 1] [0x3fffef051390] +[a33n10:140415] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n10:140415] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n10:140415] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n03:172648] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n03:172648] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n03:172648] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n03:172648] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n16:154185] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n16:154185] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n16:154185] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n18:123413] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n18:123413] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n13:133623] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n13:133623] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n06:119114] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n06:119114] [ 7] [b17n10:126611] *** Process received signal *** +[b17n10:126611] Signal: Segmentation fault (11) +[b17n10:126611] Signal code: (3) +[b17n10:126611] Failing at address: 0x14170000000000a0 +[b17n10:126611] [ 0] [0x1000000504d8] +[b17n10:126611] [ 1] [0x3ffff85d3140] +[b17n10:126611] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n10:126611] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n18:128540] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n18:128540] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n18:128540] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n18:128540] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n18:128540] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n18:128540] *** End of error message *** +[b02n10:01255] *** Process received signal *** +[b02n10:01255] Signal: Segmentation fault (11) +[b02n10:01255] Signal code: (3) +[b02n10:01255] Failing at address: 0x14170000000000a0 +[b02n10:01255] [ 0] [0x1000000504d8] +[b02n10:01255] [ 1] [0x3fffdc218ce0] +[b02n10:01255] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n10:01255] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n10:01255] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n10:01255] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n10:01255] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n10:01255] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n10:01255] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n10:01255] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n10:01255] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n10:01255] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n10:01255] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n10:01255] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n10:01255] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n10:01255] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n01:124277] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n01:124277] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n01:124277] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n01:124277] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n05:04520] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n05:04520] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n05:04520] [13] [0x1000000504d8] +[a33n11:156155] [ 1] [0x3fffe7ed1110] +[a33n11:156155] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n11:156155] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n11:156155] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n11:156155] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n02:122178] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n02:122178] *** End of error message *** +[b04n08:176175] *** Process received signal *** +[b04n08:176175] Signal: Segmentation fault (11) +[b04n08:176175] Signal code: (3) +[b04n08:176175] Failing at address: 0x14170000000000a0 +[b04n08:176175] [ 0] [0x1000000504d8] +[b04n08:176175] [ 1] [0x3fffc16c1eb0] +[b04n08:176175] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n08:176175] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n15:122082] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n10:155397] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n10:155397] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n10:155397] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n10:155397] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n10:155397] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n10:155397] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n10:155397] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n10:155397] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n10:155397] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n10:155397] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n08:127446] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n08:127446] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n08:127446] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n08:127446] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n15:139747] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n15:139747] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n14:126465] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n14:126465] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n14:126465] [ 6] [b09n13:164691] *** Process received signal *** +[b09n13:164691] Signal: Segmentation fault (11) +[b09n13:164691] Signal code: (3) +[b09n13:164691] Failing at address: 0xb170000000000a0 +[b09n13:164691] [ 0] [0x1000000504d8] +[b09n13:164691] [ 1] [0x3fffc134f960] +[b09n13:164691] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n13:164691] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n13:164691] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n13:164691] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n13:164691] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n13:164691] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n13:164691] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n13:164691] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n13:164691] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n13:164691] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n13:164691] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n13:164691] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n13:164691] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n13:164691] *** End of error message *** +[b06n09:168244] *** Process received signal *** +[b06n09:168244] Signal: Segmentation fault (11) +[b06n09:168244] Signal code: (3) +[b06n09:168244] Failing at address: 0x5170000000000a0 +[b06n09:168244] [ 0] [0x1000000504d8] +[b06n09:168244] [ 1] [0x3fffc39c61c0] +[b06n09:168244] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n09:168244] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n07:165092] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n07:165092] [ 5] [b05n11:176583] *** Process received signal *** +[b05n11:176583] Signal: Segmentation fault (11) +[b05n11:176583] Signal code: (3) +[b05n11:176583] Failing at address: 0xb170000000000a0 +[b05n11:176583] [ 0] [0x1000000504d8] +[b05n11:176583] [ 1] [0x3fffe5ad2040] +[b05n11:176583] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n11:176583] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n11:176583] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n11:176583] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n11:176583] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n11:176583] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n11:176583] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n11:176583] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n11:176583] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n11:176583] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n11:176583] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n11:176583] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n11:176583] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n11:176583] *** End of error message *** +[a02n09:160373] *** Process received signal *** +[a02n09:160373] Signal: Segmentation fault (11) +[a02n09:160373] Signal code: (3) +[a02n09:160373] Failing at address: 0x14170000000000a0 +[a02n09:160373] [ 0] [0x1000000504d8] +[a02n09:160373] [ 1] [0x3fffc943f0d0] +[a02n09:160373] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n09:160373] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n05:173318] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n05:173318] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n05:173318] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n05:173318] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n05:173318] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n10:10923] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n10:10923] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n10:10923] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n10:10923] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n10:10923] [11] [a30n18:10723] *** Process received signal *** +[a30n18:10723] Signal: Segmentation fault (11) +[a30n18:10723] Signal code: (3) +[a30n18:10723] Failing at address: 0x14170000000000a0 +[a30n18:10723] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n14:152748] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n14:152748] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n14:152748] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n14:152748] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n14:152748] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n10:169684] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n10:169684] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n10:169684] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n10:169684] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n01:121742] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n01:121742] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n01:121742] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n01:121742] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n10:120976] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n10:120976] *** End of error message *** +[b10n15:164411] *** Process received signal *** +[b10n15:164411] Signal: Segmentation fault (11) +[b10n15:164411] Signal code: (3) +[b10n15:164411] Failing at address: 0x14170000000000a0 +[b10n15:164411] [ 0] [0x1000000504d8] +[b10n15:164411] [ 1] [0x3fffce0ef480] +[b10n15:164411] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n15:164411] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n15:164411] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n15:164411] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n15:164411] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n15:164411] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n15:164411] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n15:164411] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n15:164411] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n15:164411] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n15:164411] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n15:164411] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n15:164411] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n15:164411] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n07:172876] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n07:172876] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n07:172876] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n07:172876] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n07:172876] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n07:172876] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n07:172876] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n07:172876] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n07:172876] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n07:172876] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n07:172876] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n03:09641] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n03:09641] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n03:09641] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n03:09641] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n03:09641] [11] [a31n18:169362] *** Process received signal *** +[a31n18:169362] Signal: Segmentation fault (11) +[a31n18:169362] Signal code: (3) +[a31n18:169362] Failing at address: 0x14170000000000a0 +[a31n18:169362] [ 0] [0x1000000504d8] +[a31n18:169362] [ 1] [0x3fffe208c9b0] +[a31n18:169362] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n18:169362] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n18:169362] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n18:169362] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n18:169362] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n18:169362] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n18:169362] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n18:169362] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n18:169362] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n18:169362] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n18:169362] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n18:169362] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n18:169362] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n18:169362] *** End of error message *** +[a26n02:74428] *** Process received signal *** +[a26n02:74428] Signal: Segmentation fault (11) +[a26n02:74428] Signal code: (3) +[a26n02:74428] Failing at address: 0x14170000000000a0 +[a26n02:74428] [ 0] [0x1000000504d8] +[a26n02:74428] [ 1] [0x3fffe3f83c00] +[a26n02:74428] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n02:74428] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n02:74428] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n02:74428] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n02:74428] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n02:74428] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n02:74428] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n02:74428] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n02:74428] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n02:74428] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n02:74428] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n02:74428] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n10:135852] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n10:135852] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n10:135852] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n10:135852] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n02:125267] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n02:125267] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n02:125267] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n02:125267] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n02:125267] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n02:125267] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n02:125267] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n02:125267] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n02:125267] *** End of error message *** +[a31n14:165664] *** Process received signal *** +[a31n14:165664] Signal: Segmentation fault (11) +[a31n14:165664] Signal code: (3) +[a31n14:165664] Failing at address: 0x14170000000000a0 +[a31n14:165664] [ 0] [0x1000000504d8] +[a31n14:165664] [ 1] [0x3fffe784a520] +[a31n14:165664] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n14:165664] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n14:165664] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n14:165664] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n14:165664] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n14:165664] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n14:165664] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n14:165664] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n14:165664] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n14:165664] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n14:165664] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n14:165664] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n14:165664] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n14:165664] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n04:06833] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n04:06833] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n04:06833] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n04:06833] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n04:06833] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n17:122356] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n17:122356] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n17:122356] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n17:122356] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n17:122356] [14] [a31n02:158420] Signal: Segmentation fault (11) +[a31n02:158420] Signal code: (3) +[a31n02:158420] Failing at address: 0x14170000000000a0 +[a31n02:158420] [ 0] [0x1000000504d8] +[a31n02:158420] [ 1] [0x3ffffa2c90e0] +[a31n02:158420] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n02:158420] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n02:158420] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n02:158420] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n02:158420] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n02:158420] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n02:158420] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n02:158420] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n02:158420] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n02:158420] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n02:158420] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n02:158420] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n02:158420] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n02:158420] *** End of error message *** +[b06n03:164494] *** Process received signal *** +[b06n03:164494] Signal: Segmentation fault (11) +[b06n03:164494] Signal code: (3) +[b06n03:164494] Failing at address: 0x14170000000000a0 +[b06n03:164494] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n12:122872] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n04:130749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n04:130749] *** End of error message *** +[b18n16:124961] *** Process received signal *** +[b18n16:124961] Signal: Segmentation fault (11) +[b18n16:124961] Signal code: (3) +[b18n16:124961] Failing at address: 0x14170000000000a0 +[b18n16:124961] [ 0] [0x1000000504d8] +[b18n16:124961] [ 1] [0x3fffde240060] +[b18n16:124961] [0x3fffe7064350] +[a27n07:45306] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n07:45306] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n07:45306] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n13:155850] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n13:155850] [ 4] [b07n02:174055] *** Process received signal *** +[b07n02:174055] Signal: Segmentation fault (11) +[b07n02:174055] Signal code: (3) +[b07n02:174055] Failing at address: 0xb170000000000a0 +[b07n02:174055] [ 0] [0x1000000504d8] +[b07n02:174055] [ 1] [0x3fffc91608a0] +[b07n02:174055] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n02:174055] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n02:174055] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n02:174055] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n02:174055] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n02:174055] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n02:174055] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n02:174055] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n02:174055] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n02:174055] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n02:174055] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n02:174055] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n02:174055] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n02:174055] *** End of error message *** +[b09n17:163177] *** Process received signal *** +[b09n17:163177] Signal: Segmentation fault (11) +[b09n17:163177] Signal code: (3) +[b09n17:163177] Failing at address: 0xb170000000000a0 +[b09n17:163177] [ 0] [0x1000000504d8] +[b09n17:163177] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n05:149465] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n05:149465] [ 5] [0x3fffc4c16190] +[b05n14:177510] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n14:177510] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n14:177510] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n14:177510] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n14:177510] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n14:177510] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n14:177510] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n14:177510] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n14:177510] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n14:177510] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n14:177510] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n14:177510] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n14:177510] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n14:177510] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n12:160189] *** End of error message *** +[b05n09:106786] *** Process received signal *** +[b05n09:106786] Signal: Segmentation fault (11) +[b05n09:106786] Signal code: (3) +[b05n09:106786] Failing at address: 0xb170000000000a0 +[b05n09:106786] [ 0] [0x1000000504d8] +[b05n09:106786] [ 1] [0x3fffd2b65b40] +[b05n09:106786] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n09:106786] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n09:106786] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n09:170657] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n09:170657] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n03:135036] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n03:135036] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n05:176910] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n05:176910] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n05:176910] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n05:176910] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n05:176910] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n10:122757] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n02:125370] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n02:125370] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n02:125370] [14] [a02n08:146116] *** Process received signal *** +[a02n08:146116] Signal: Segmentation fault (11) +[a02n08:146116] Signal code: (3) +[a02n08:146116] Failing at address: 0x14170000000000a0 +[a02n08:146116] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n04:129407] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n04:129407] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n04:129407] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n04:129407] *** End of error message *** +[b17n09:128492] *** Process received signal *** +[b17n09:128492] Signal: Segmentation fault (11) +[b17n09:128492] Signal code: (3) +[b17n09:128492] Failing at address: 0xb170000000000a0 +[b17n09:128492] [ 0] [0x1000000504d8] +[b17n09:128492] [ 1] [0x3ffffecf11f0] +[b17n09:128492] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n09:128492] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n07:156486] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n07:156486] *** End of error message *** +[0x3ffff80fbfb0] +[b17n14:126118] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n14:126118] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n14:126118] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n14:126118] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n07:02841] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n07:02841] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n07:02841] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n07:02841] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n07:02841] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n08:118553] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n08:118553] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n08:118553] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n08:118553] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n08:118553] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n08:118553] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n04:156413] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n04:156413] [ 6] [b18n04:136997] *** Process received signal *** +[b18n04:136997] Signal: Segmentation fault (11) +[b18n04:136997] Signal code: (3) +[b18n04:136997] Failing at address: 0xb170000000000a0 +[b18n04:136997] [ 0] [0x1000000504d8] +[b18n04:136997] [ 1] [0x3fffd84da240] +[b18n04:136997] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n04:136997] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n04:136997] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n15:165474] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n15:165474] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n15:165474] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n15:165474] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n15:165474] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n10:176261] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n10:176261] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n11:128954] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n11:128954] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n11:128954] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n03:147537] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n03:147537] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n03:147537] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n03:147537] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n03:147537] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n03:147537] [14] [0x3fffff013870] +[a26n03:68831] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n03:68831] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n03:68831] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n03:68831] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n03:68831] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n03:68831] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n03:68831] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n11:134748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n11:134748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n11:134748] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n11:134748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n11:134748] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n07:163253] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n07:163253] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n07:163253] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n07:163253] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n07:163253] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n08:165282] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n08:165282] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n08:165282] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n08:165282] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n13:137744] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n13:137744] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n13:137744] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n13:137744] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n14:122444] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n14:122444] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n14:122444] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n14:122444] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n14:122444] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n18:123866] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n18:123866] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n18:123866] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n18:123866] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n18:123866] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n18:123866] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n01:123574] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n01:123574] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n03:119829] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n03:119829] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n08:124119] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n08:124119] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n08:124119] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n08:124119] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n08:124119] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n08:124119] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n05:173823] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n05:173823] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n05:173823] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n05:173823] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n03:126186] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n03:126186] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n03:126186] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n03:126186] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n07:125450] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n07:125450] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n12:145262] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n12:145262] *** End of error message *** +[b01n02:01431] *** Process received signal *** +[b01n02:01431] Signal: Segmentation fault (11) +[b01n02:01431] Signal code: (3) +[b01n02:01431] Failing at address: 0x14170000000000a0 +[b01n02:01431] [ 0] [0x1000000504d8] +[b01n02:01431] [ 1] [0x3ffffacdd450] +[b01n02:01431] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n02:01431] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n02:01431] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n02:01431] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n02:01431] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n02:01431] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n02:01431] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n02:01431] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n02:01431] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n02:01431] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n02:01431] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n02:01431] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n02:01431] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n02:01431] *** End of error message *** +[0x1000000504d8] +[b12n16:129236] [ 1] [0x3fffc6f717b0] +[b12n16:129236] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n16:129236] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n16:129236] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n03:124682] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n03:124682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n03:124682] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n06:148310] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n06:148310] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n06:148310] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n13:165386] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n13:165386] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n13:165386] [ 6] [b09n04:177597] *** Process received signal *** +[b09n04:177597] Signal: Segmentation fault (11) +[b09n04:177597] Signal code: (3) +[b09n04:177597] Failing at address: 0x14170000000000a0 +[b09n04:177597] [ 0] [0x1000000504d8] +[b09n04:177597] [ 1] [0x3fffeca0d8d0] +[b09n04:177597] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n04:177597] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n12:120405] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n12:120405] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n07:156139] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n07:156139] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n14:177224] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n14:177224] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n14:177224] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n13:177638] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n13:177638] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n13:177638] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n13:177638] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n14:116411] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n14:116411] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n14:116411] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n18:121303] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n18:121303] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n05:05607] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n05:05607] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n05:05607] [13] [0x3fffc0cfa730] +[b17n04:136608] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n04:136608] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n04:136608] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n04:136608] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n03:155728] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n03:155728] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n03:155728] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n03:155728] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n03:155728] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n09:166573] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n09:166573] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n09:143901] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n09:143901] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n09:143901] *** End of error message *** +[0x3fffc06ce9a0] +[a27n05:53372] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n05:53372] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n05:53372] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n05:53372] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n12:177497] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n12:177497] *** End of error message *** +[a33n07:149475] *** Process received signal *** +[a33n07:149475] Signal: Segmentation fault (11) +[a33n07:149475] Signal code: (3) +[a33n07:149475] Failing at address: 0xb170000000000a0 +[a33n07:149475] [ 0] [0x1000000504d8] +[a33n07:149475] [ 1] [0x3fffd73e2430] +[a33n07:149475] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n04:117354] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n04:117354] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n04:117354] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n04:117354] *** End of error message *** +[a33n12:168641] *** Process received signal *** +[a33n12:168641] Signal: Segmentation fault (11) +[a33n12:168641] Signal code: (3) +[a33n12:168641] Failing at address: 0xb170000000000a0 +[a33n12:168641] [ 0] [0x1000000504d8] +[b13n08:144854] [ 1] [0x3ffffeb42bf0] +[b13n08:144854] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n08:144854] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n08:144854] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n16:168050] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n13:54241] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n13:54241] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n13:54241] [ 7] [0x3ffffcfcc380] +[a32n16:147286] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n16:147286] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n16:147286] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n16:147286] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n05:121826] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n05:121826] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n05:121826] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n08:138175] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n08:138175] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n08:138175] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n08:138175] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n12:119805] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n12:119805] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n12:119805] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n12:119805] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n12:119805] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n12:119805] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n18:03311] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n18:03311] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n04:168490] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n04:168490] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n04:168490] *** End of error message *** +[b12n14:139845] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n14:120639] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n14:120639] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n14:120639] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n14:120639] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n14:120639] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n18:122556] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n18:122556] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n18:122556] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n18:122556] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n17:125809] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n17:125809] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n17:125809] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n17:125809] [11] [b02n12:180113] *** Process received signal *** +[b02n12:180113] Signal: Segmentation fault (11) +[b02n12:180113] Signal code: (3) +[b02n12:180113] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n10:126193] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n10:126193] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n16:130375] *** End of error message *** +[b09n07:173040] *** Process received signal *** +[b09n07:173040] Signal: Segmentation fault (11) +[b09n07:173040] Signal code: (3) +[b09n07:173040] Failing at address: 0x14170000000000a0 +[b09n07:173040] [ 0] [0x1000000504d8] +[b09n07:173040] [ 1] [0x3ffffa2f2630] +[b09n07:173040] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n07:173040] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n07:173040] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n07:173040] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n07:173040] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n07:173040] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n07:173040] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n07:173040] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n07:173040] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n07:173040] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n07:173040] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n07:173040] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n07:173040] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n07:173040] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n13:138091] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n13:138091] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n13:138091] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n10:124919] *** Process received signal *** +[b14n10:124919] Signal: Segmentation fault (11) +[b14n10:124919] Signal code: (3) +[b14n10:124919] Failing at address: 0xb170000000000a0 +[b14n10:124919] [ 0] [0x1000000504d8] +[b14n10:124919] [ 1] [0x3fffe60e7e60] +[b14n10:124919] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n10:124919] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n10:124919] [ 4] [b17n11:124446] *** Process received signal *** +[b17n11:124446] Signal: Segmentation fault (11) +[b17n11:124446] Signal code: (3) +[b17n11:124446] Failing at address: 0xb170000000000a0 +[b17n11:124446] [ 0] [0x1000000504d8] +[b17n11:124446] [ 1] [0x3ffffefee6e0] +[b17n11:124446] [ 2] [a33n14:136135] *** Process received signal *** +[a33n14:136135] Signal: Segmentation fault (11) +[a33n14:136135] Signal code: (3) +[a33n14:136135] Failing at address: 0x14170000000000a0 +[a33n14:136135] [ 0] [0x1000000504d8] +[a33n14:136135] [ 1] [0x3ffff239c9a0] +[a33n14:136135] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n14:136135] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n10:141233] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n10:141233] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n10:141233] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n10:141233] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n05:173849] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n05:173849] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n05:173849] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n05:173849] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n05:173849] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n05:173849] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n05:173849] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n05:173849] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n05:173849] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n05:173849] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n12:164381] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n15:120058] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n15:120058] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n15:120058] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n15:120058] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n15:120058] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n01:163025] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n15:148166] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n15:148166] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n15:148166] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n15:148166] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n15:148166] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n18:176808] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n18:176808] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n18:176808] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n18:176808] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n14:150134] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n14:150134] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n14:150134] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n14:150134] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n14:150134] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n14:150134] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n14:150134] [13] [b21n15:119255] *** Process received signal *** +[b21n15:119255] Signal: Segmentation fault (11) +[b21n15:119255] Signal code: (3) +[b21n15:119255] Failing at address: 0x14170000000000a0 +[b21n15:119255] [ 0] [0x1000000504d8] +[b21n15:119255] [ 1] [0x3fffdedcba40] +[b21n15:119255] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n15:119255] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n15:119255] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n15:119255] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n15:119255] [ 6] [a32n04:153963] *** Process received signal *** +[a32n04:153963] Signal: Segmentation fault (11) +[a32n04:153963] Signal code: (3) +[a32n04:153963] Failing at address: 0xb170000000000a0 +[a32n04:153963] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n14:119488] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n09:173061] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n09:173061] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n07:174562] *** End of error message *** +[a29n11:162472] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n10:126791] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n10:126791] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n10:126791] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n10:126791] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n10:126791] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n10:126791] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n10:126791] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n10:126791] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n10:126791] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n10:126791] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n10:126791] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n10:126791] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n02:177547] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n02:177547] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n02:177547] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n02:177547] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n13:117510] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n17:168160] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n17:168160] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n13:166926] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n13:166926] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n06:164014] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n06:164014] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n06:164014] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n06:164014] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n06:164014] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n02:119533] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n02:119533] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n11:163259] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n11:163259] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n11:163259] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n11:163259] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n11:163259] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n15:151251] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n15:151251] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n15:151251] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n15:151251] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n15:151251] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n10:134732] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n10:134732] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n12:178718] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n12:178718] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n12:178718] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n12:178718] *** End of error message *** +[b20n04:126345] Signal: Segmentation fault (11) +[b20n04:126345] Signal code: (3) +[b20n04:126345] Failing at address: 0xb170000000000a0 +[b20n04:126345] [ 0] [0x1000000504d8] +[b20n04:126345] [ 1] [0x3fffec3915d0] +[b20n04:126345] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n04:126345] [ 3] [b09n16:173550] *** Process received signal *** +[b09n16:173550] Signal: Segmentation fault (11) +[b09n16:173550] Signal code: (3) +[b09n16:173550] Failing at address: 0xb170000000000a0 +[b17n05:125613] *** Process received signal *** +[b17n05:125613] Signal: Segmentation fault (11) +[b17n05:125613] Signal code: (3) +[b17n05:125613] Failing at address: 0xb170000000000a0 +[b17n05:125613] [ 0] [0x1000000504d8] +[b17n05:125613] [ 1] [0x3ffffe7d31e0] +[b17n05:125613] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n05:125613] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n18:08428] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n18:08428] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n18:08428] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n18:08428] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n18:164796] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n18:164796] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n03:158656] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n03:158656] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n03:158656] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n03:158656] [ 8] [0x1000000504d8] +[a31n11:153416] [ 1] [0x3fffda4fdd50] +[a31n11:153416] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n11:153416] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n11:153416] [ 4] [b19n13:123885] *** Process received signal *** +[b19n13:123885] Signal: Segmentation fault (11) +[b19n13:123885] Signal code: (3) +[b19n13:123885] Failing at address: 0xb170000000000a0 +[b19n13:123885] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n04:179174] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n04:179174] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n04:179174] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n04:179174] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n04:179174] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n04:179174] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n04:179174] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n11:124434] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n11:124434] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n10:162610] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n10:162610] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n10:162610] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n10:162610] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n10:162610] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n10:162610] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n10:162610] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n16:149062] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n16:149062] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n12:120130] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n12:120130] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n12:120130] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n12:120130] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n12:120130] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n17:129933] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n17:129933] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n17:129933] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n17:129933] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n17:129933] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n17:129933] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n17:129933] [13] [b01n10:173678] *** Process received signal *** +[b01n10:173678] Signal: Segmentation fault (11) +[b01n10:173678] Signal code: (3) +[b01n10:173678] Failing at address: 0xb170000000000a0 +[b01n10:173678] [ 0] [0x1000000504d8] +[b01n10:173678] [ 1] [0x3fffcefc9c90] +[b01n10:173678] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n10:173678] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n10:173678] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n10:173678] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n10:173678] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n10:173678] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n10:173678] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n10:173678] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n10:173678] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n10:173678] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n10:173678] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n10:173678] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n10:173678] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n10:173678] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n09:127999] *** End of error message *** +[0x3fffdcfe5720] +[b19n15:124800] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n15:124800] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n15:124800] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n15:124800] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n13:168161] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n13:168161] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n13:168161] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n09:137775] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n09:137775] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n09:137775] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n09:137775] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n09:137775] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n18:140384] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n18:140384] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n18:140384] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n18:140384] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n18:140384] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n18:140384] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n18:131585] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n18:131585] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n05:164943] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n09:120693] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n08:54626] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n08:54626] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n08:54626] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n08:54626] [ 8] [0x3fffcdf191c0] +[b14n06:127276] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n06:127276] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n06:127276] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n06:127276] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n06:127276] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n17:171198] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n17:171198] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n17:171198] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n17:171198] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n02:176051] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n02:176051] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n02:176051] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n09:161231] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n09:161231] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n09:161231] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n09:161231] [10] [0x1000000504d8] +[b13n16:128438] [ 1] [0x3fffc83f4300] +[b13n16:128438] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n16:128438] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n16:128438] [ 4] [0x1000000504d8] +[a33n02:167599] [ 1] [0x3ffff790c800] +[a33n02:167599] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n02:167599] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n02:167599] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n02:167599] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n08:116171] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n08:116171] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n08:116171] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n08:116171] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n04:166208] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n04:166208] *** End of error message *** +[b18n08:131379] *** Process received signal *** +[b18n08:131379] Signal: Segmentation fault (11) +[b18n08:131379] Signal code: (3) +[b18n08:131379] Failing at address: 0x14170000000000a0 +[b18n08:131379] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n06:167368] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n06:167368] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n06:167368] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n07:117359] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n07:117359] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n16:165920] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n16:165920] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n07:133372] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n07:133372] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n07:133372] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n07:133372] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n14:133065] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n14:133065] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n14:133065] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n14:133065] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n08:76223] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n08:76223] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n17:124166] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n17:124166] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n17:124166] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n17:124166] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n17:124166] *** End of error message *** +[a02n04:41671] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n16:127352] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n16:127352] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n16:127352] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n16:127352] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n18:07422] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n18:07422] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n18:07422] [ 8] [a02n18:21772] *** Process received signal *** +[a02n18:21772] Signal: Segmentation fault (11) +[a02n18:21772] Signal code: (3) +[a02n18:21772] Failing at address: 0x14170000000000a0 +[a02n18:21772] [ 0] [0x1000000504d8] +[a02n18:21772] [ 1] [0x3fffc4469280] +[a02n18:21772] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n18:21772] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n18:21772] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n18:21772] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n18:21772] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n18:21772] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n18:21772] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n18:21772] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n18:21772] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n18:21772] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n18:21772] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n18:21772] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n18:21772] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n18:21772] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n12:132099] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n04:52706] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n04:52706] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n04:52706] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n04:52706] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n04:52706] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n11:23062] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n11:23062] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n18:69155] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n18:69155] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n18:69155] *** End of error message *** +[a02n03:38502] *** Process received signal *** +[a02n03:38502] Signal: Segmentation fault (11) +[a02n03:38502] Signal code: (3) +[a02n03:38502] Failing at address: 0x14170000000000a0 +[a02n03:38502] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n14:176070] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n14:176070] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n14:176070] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n14:176070] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n08:172899] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n08:172899] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n08:172899] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n13:122033] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n13:122033] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n13:122033] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n13:122033] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n13:122033] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n01:48805] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n01:48805] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n01:128829] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n01:128829] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n01:128829] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n07:180193] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n07:180193] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n07:180193] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n07:180193] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n07:180193] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n07:180193] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n07:180193] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n15:177059] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n15:177059] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n01:166960] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n01:166960] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n01:166960] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n01:166960] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n11:133970] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n11:133970] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n11:133970] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n11:133970] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n11:133970] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n11:133970] [14] [0x1000000504d8] +[b19n03:123614] [ 1] [0x3fffdeb06450] +[b19n03:123614] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n03:123614] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n03:123614] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n03:123614] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n14:168374] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n14:168374] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n14:168374] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n16:135791] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n16:135791] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n09:149099] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n09:149099] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n09:149099] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n09:149099] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n12:167678] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n11:170898] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n11:170898] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n11:170898] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n11:170898] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n11:170898] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n11:170898] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n11:170898] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n11:170898] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n11:170898] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n11:170898] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n11:170898] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n11:170898] *** End of error message *** +[a27n09:51333] *** Process received signal *** +[a27n09:51333] Signal: Segmentation fault (11) +[a27n09:51333] Signal code: (3) +[a27n09:51333] Failing at address: 0x14170000000000a0 +[a27n09:51333] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n12:12764] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n12:12764] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n11:160347] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n11:160347] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n11:160347] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n11:160347] [ 8] [b19n06:126379] *** Process received signal *** +[b19n06:126379] Signal: Segmentation fault (11) +[b19n06:126379] Signal code: (3) +[b19n06:126379] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n13:114398] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n13:114398] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n13:114398] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n17:179116] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n17:179116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n17:179116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n17:179116] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n17:179116] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n17:179116] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n17:179116] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n17:179116] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n17:179116] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n16:116525] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n16:116525] *** End of error message *** +[a34n17:149907] *** Process received signal *** +[a34n17:149907] Signal: Segmentation fault (11) +[a34n17:149907] Signal code: (3) +[a34n17:149907] Failing at address: 0xb170000000000a0 +[a34n17:149907] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n10:142930] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n10:142930] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n10:142930] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n10:142930] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n10:142930] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n10:142930] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n15:155433] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n15:155433] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n15:155433] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n15:155433] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n15:155433] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n15:155433] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n15:155433] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n15:155433] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n05:135166] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n05:135166] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n05:135166] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n05:135166] [10] [a31n05:158575] *** Process received signal *** +[a31n05:158575] Signal: Segmentation fault (11) +[a31n05:158575] Signal code: (3) +[a31n05:158575] Failing at address: 0x14170000000000a0 +[a31n05:158575] [ 0] [0x1000000504d8] +[a31n05:158575] [ 1] [0x3fffd9c48a30] +[a31n05:158575] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n05:158575] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n05:158575] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n05:158575] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n05:158575] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n05:158575] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n05:158575] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n05:158575] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n05:158575] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n05:158575] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n05:158575] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n05:158575] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n05:158575] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n05:158575] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n16:157888] *** End of error message *** +[b22n08:118578] *** Process received signal *** +[b22n08:118578] Signal: Segmentation fault (11) +[b22n08:118578] Signal code: (3) +[b22n08:118578] Failing at address: 0x14170000000000a0 +[b22n08:118578] [ 0] [0x1000000504d8] +[b22n08:118578] [ 1] [0x3ffff4676740] +[b22n08:118578] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n08:118578] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n08:118578] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n08:118578] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n08:118578] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n08:118578] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n08:118578] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n08:118578] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n08:118578] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n08:118578] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n08:118578] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n08:118578] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n08:118578] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n08:118578] *** End of error message *** +[0x3ffff1079310] +[b19n09:123418] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n09:123418] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n09:123418] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n09:123418] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n09:123418] [ 6] [b03n02:06562] *** Process received signal *** +[b03n02:06562] Signal: Segmentation fault (11) +[b03n02:06562] Signal code: (3) +[b03n02:06562] Failing at address: 0x14170000000000a0 +[b03n02:06562] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n07:118166] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n07:118166] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n07:118166] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n07:118166] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n18:179700] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n18:179700] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n18:179700] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n10:176310] *** Process received signal *** +[a28n10:176310] Signal: Segmentation fault (11) +[a28n10:176310] Signal code: (3) +[a28n10:176310] Failing at address: 0x14170000000000a0 +[a28n10:176310] [ 0] [0x1000000504d8] +[a28n10:176310] [ 1] [0x3ffffa357df0] +[a28n10:176310] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n10:176310] [ 3] [b02n17:03843] [ 0] [0x1000000504d8] +[b02n17:03843] [ 1] [0x3fffcad95540] +[b02n17:03843] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n17:03843] [ 3] [b14n13:122139] *** Process received signal *** +[b14n13:122139] Signal: Segmentation fault (11) +[b14n13:122139] Signal code: (3) +[b14n13:122139] Failing at address: 0x14170000000000a0 +[b14n13:122139] [ 0] [0x1000000504d8] +[b14n13:122139] [ 1] [0x3fffdb942720] +[b14n13:122139] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n13:122139] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n13:122139] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n13:122139] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n13:122139] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n13:122139] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n13:122139] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n13:122139] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n13:122139] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n13:122139] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n13:122139] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n13:122139] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n13:122139] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n13:122139] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n14:133632] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n01:167297] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n01:167297] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n12:122963] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n12:122963] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n12:122963] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n12:122963] [13] [0x3ffff292eff0] +[a28n09:03926] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n09:03926] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n09:03926] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n09:03926] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n14:169522] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n13:176366] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n13:176366] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n13:176366] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n13:176366] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n17:15874] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n17:15874] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n17:15874] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n17:15874] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n01:141818] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n01:141818] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n01:141818] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n01:141818] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n06:01841] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n06:01841] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n06:01841] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n06:01841] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n14:128870] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n14:128870] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n14:128870] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n14:128870] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n14:128870] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n17:138142] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n17:138142] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n17:138142] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n17:138142] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n17:138142] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n07:138202] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n07:138202] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n07:138202] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n03:177927] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n03:177927] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n03:177927] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n03:177927] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n03:177927] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n07:132089] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n07:132089] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n07:132089] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n08:08956] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n08:08956] [ 5] [a36n06:143009] *** Process received signal *** +[a36n06:143009] Signal: Segmentation fault (11) +[a36n06:143009] Signal code: (3) +[a36n06:143009] Failing at address: 0xb170000000000a0 +[a36n06:143009] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n16:124665] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n16:124665] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n16:124665] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n16:124665] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n16:124665] *** End of error message *** +[b09n14:159238] *** Process received signal *** +[b09n14:159238] Signal: Segmentation fault (11) +[b09n14:159238] Signal code: (3) +[b09n14:159238] Failing at address: 0x5170000000000a0 +[b09n14:159238] [ 0] [0x1000000504d8] +[b09n14:159238] [ 1] [0x3fffc66b2a90] +[b09n14:159238] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n14:159238] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n02:166746] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n02:166746] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n02:166746] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n06:43018] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n06:43018] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n06:43018] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n14:136662] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n14:136662] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n14:136662] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n14:136662] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n01:128157] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n01:128157] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n01:128157] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n18:126904] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n18:126904] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n18:126904] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n18:126904] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n04:169912] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n04:169912] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n04:169912] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n14:175715] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n14:175715] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n14:175715] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n14:175715] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n14:175715] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n02:164488] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n02:164488] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n10:67857] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n10:67857] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n10:67857] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n10:67857] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n07:153177] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n07:153177] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n07:153177] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n07:153177] *** End of error message *** +[b19n07:131672] *** Process received signal *** +[b19n07:131672] Signal: Segmentation fault (11) +[b19n07:131672] Signal code: (3) +[b19n07:131672] Failing at address: 0x14170000000000a0 +[b19n07:131672] [ 0] [0x1000000504d8] +[b19n07:131672] [ 1] [0x3fffe3d8f950] +[b19n07:131672] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n03:139012] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n03:139012] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n03:139012] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n03:139012] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n03:139012] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n03:139012] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n03:139012] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n03:139012] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n03:139012] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n03:139012] *** End of error message *** +[0x3fffc9127ab0] +[b09n11:171964] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n11:171964] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n11:171964] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n11:171964] [ 5] [a29n02:163028] *** Process received signal *** +[a29n02:163028] Signal: Segmentation fault (11) +[a29n02:163028] Signal code: (3) +[a29n02:163028] Failing at address: 0x5170000000000a0 +[a29n02:163028] [ 0] [0x1000000504d8] +[a29n02:163028] [ 1] [0x3fffe6c5b210] +[a29n02:163028] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n02:163028] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n13:123709] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n13:123709] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n13:123709] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n08:147823] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n08:147823] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n08:147823] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n08:147823] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n13:156921] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n03:10292] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n03:10292] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n03:10292] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n03:10292] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n03:10292] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n11:152986] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n11:152986] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n11:152986] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n11:152986] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n11:152986] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n11:152986] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n11:152986] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n11:152986] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n11:152986] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n11:152986] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n17:156181] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n17:156181] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n17:156181] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n17:156181] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n17:156181] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n17:156181] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n17:156181] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n17:150928] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n17:150928] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n17:150928] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n09:121755] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n09:121755] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n09:121755] [11] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n17:164293] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n17:164293] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n17:164293] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n17:164293] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n17:164293] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n17:164293] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n17:164293] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n17:164293] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n17:164293] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n17:164293] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n16:01571] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n16:01571] *** End of error message *** +[b03n15:85943] *** Process received signal *** +[b03n15:85943] Signal: Segmentation fault (11) +[b03n15:85943] Signal code: (3) +[b03n15:85943] Failing at address: 0x5170000000000a0 +[b03n15:85943] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n11:137873] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n11:137873] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n11:137873] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n11:137873] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n11:137873] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n11:152842] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n11:152842] [ 5] [0x3fffe54bc800] +[a27n18:28217] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n18:28217] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n18:28217] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n18:28217] [ 5] [b05n12:11144] *** Process received signal *** +[b05n12:11144] Signal: Segmentation fault (11) +[b05n12:11144] Signal code: (3) +[b05n12:11144] Failing at address: 0xb170000000000a0 +[b05n12:11144] [ 0] [0x1000000504d8] +[b05n12:11144] [ 1] [0x3ffff9a0a960] +[b05n12:11144] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n12:11144] [ 3] [b20n08:124722] *** Process received signal *** +[b20n08:124722] Signal: Segmentation fault (11) +[b20n08:124722] Signal code: (3) +[b20n08:124722] Failing at address: 0xb170000000000a0 +[b20n08:124722] [ 0] [0x1000000504d8] +[b20n08:124722] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n09:173749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n09:173749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n13:01300] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n13:01300] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n13:01300] [ 9] [b21n10:130580] *** Process received signal *** +[b21n10:130580] Signal: Segmentation fault (11) +[b21n10:130580] Signal code: (3) +[b21n10:130580] Failing at address: 0xb170000000000a0 +[b21n10:130580] [ 0] [b12n02:143966] *** Process received signal *** +[b12n02:143966] Signal: Segmentation fault (11) +[b12n02:143966] Signal code: (3) +[b12n02:143966] Failing at address: 0x5170000000000a0 +[b12n02:143966] [ 0] [0x1000000504d8] +[b12n02:143966] [ 1] [b06n08:174408] *** Process received signal *** +[b06n08:174408] Signal: Segmentation fault (11) +[b06n08:174408] Signal code: (3) +[b06n08:174408] Failing at address: 0x5170000000000a0 +[b06n08:174408] [ 0] [0x1000000504d8] +[b06n08:174408] [ 1] [0x3ffff3289dd0] +[b06n08:174408] [ 2] [b21n01:130408] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n14:161618] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n14:161618] [ 5] [a30n16:153291] *** Process received signal *** +[a30n16:153291] Signal: Segmentation fault (11) +[a30n16:153291] Signal code: (3) +[a30n16:153291] Failing at address: 0x14170000000000a0 +[a30n16:153291] [ 0] [b21n06:122163] *** Process received signal *** +[b21n06:122163] Signal: Segmentation fault (11) +[b21n06:122163] Signal code: (3) +[b21n06:122163] Failing at address: 0xb170000000000a0 +[b21n06:122163] [ 0] [0x1000000504d8] +[b21n06:122163] [ 1] [a28n11:01171] *** Process received signal *** +[a28n11:01171] Signal: Segmentation fault (11) +[a28n11:01171] Signal code: (3) +[a28n11:01171] Failing at address: 0xb170000000000a0 +[a28n11:01171] [ 0] [0x1000000504d8] +[a28n11:01171] [ 1] [b14n09:119545] *** Process received signal *** +[b14n09:119545] Signal: Segmentation fault (11) +[b14n09:119545] Signal code: (3) +[b14n09:119545] Failing at address: 0xb170000000000a0 +[b14n09:119545] [ 0] [b08n16:171915] *** Process received signal *** +[b08n16:171915] Signal: Segmentation fault (11) +[b08n16:171915] Signal code: (3) +[b08n16:171915] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n16:122495] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n16:122495] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n16:122495] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n16:122495] [ 9] [b18n09:123813] *** Process received signal *** +[b18n09:123813] Signal: Segmentation fault (11) +[b18n09:123813] Signal code: (3) +[b18n09:123813] Failing at address: 0xb170000000000a0 +[b18n09:123813] [ 0] [0x1000000504d8] +[b18n09:123813] [ 1] [0x3fffcc0a11f0] +[b18n09:123813] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n09:123813] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n09:123813] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n09:123813] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n09:123813] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n09:123813] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n09:123813] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n09:123813] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n09:123813] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n09:123813] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n09:123813] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n09:123813] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n09:123813] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n09:123813] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n01:173778] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n01:173778] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n15:177861] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n15:177861] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n15:177861] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n15:177861] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n15:177861] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n15:177861] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n15:177861] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n12:148381] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n12:148381] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n12:148381] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n12:148381] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n12:148381] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n12:123233] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n12:123233] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n12:123233] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n12:123233] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n12:123233] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n12:123233] [14] [b21n03:127629] *** Process received signal *** +[b21n03:127629] Signal: Segmentation fault (11) +[b21n03:127629] Signal code: (3) +[b21n03:127629] Failing at address: 0xb170000000000a0 +[b21n03:127629] [ 0] [0x1000000504d8] +[b21n03:127629] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n17:119582] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n17:119582] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n17:119582] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n14:173896] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n14:173896] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n14:173896] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n14:173896] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n14:173896] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n12:156918] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n12:156918] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n02:120190] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n02:120190] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n08:129889] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n08:129889] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n08:129889] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n08:129889] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n08:129889] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n15:148579] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n15:148579] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n04:138158] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n04:138158] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n04:138158] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n02:163322] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n02:163322] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n02:163322] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n02:163322] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n17:164560] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n17:164560] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n17:164560] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n10:122209] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n10:122209] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n10:122209] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n10:122209] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n10:122209] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n10:122209] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n10:122209] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n10:122209] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n10:122209] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n10:122209] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n12:52453] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n12:52453] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n12:52453] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n12:52453] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n12:52453] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n12:52453] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n12:52453] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n18:163107] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n18:163107] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n18:163107] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n18:163107] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n09:122068] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n09:122068] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n09:122068] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n09:122068] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n01:145734] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n01:145734] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n04:175999] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n04:175999] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n04:175999] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n04:175999] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n04:175999] [14] [b17n07:125474] *** Process received signal *** +[b17n07:125474] Signal: Segmentation fault (11) +[b17n07:125474] Signal code: (3) +[b17n07:125474] Failing at address: 0x5170000000000a0 +[b17n07:125474] [ 0] [0x1000000504d8] +[b17n07:125474] [ 1] [0x3fffe43334a0] +[b17n07:125474] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n07:125474] [ 3] [a32n13:163195] Signal: Segmentation fault (11) +[a32n13:163195] Signal code: (3) +[a32n13:163195] Failing at address: 0x14170000000000a0 +[a32n13:163195] [ 0] [0x1000000504d8] +[a32n13:163195] [ 1] [0x3fffd543f5c0] +[a32n13:163195] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n13:163195] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n04:164046] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n04:164046] [ 6] [b09n03:168209] *** Process received signal *** +[b09n03:168209] Signal: Segmentation fault (11) +[b09n03:168209] Signal code: (3) +[b09n03:168209] Failing at address: 0x14170000000000a0 +[b09n03:168209] [ 0] [0x1000000504d8] +[b09n03:168209] [ 1] [0x3ffff72701c0] +[b09n03:168209] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n03:168209] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n08:163161] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n08:163161] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n08:163161] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n08:163161] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n08:163161] [13] [b04n11:176298] *** Process received signal *** +[b04n11:176298] Signal: Segmentation fault (11) +[b04n11:176298] Signal code: (3) +[b04n11:176298] Failing at address: 0xb170000000000a0 +[b04n11:176298] [ 0] [0x1000000504d8] +[b04n11:176298] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n07:127355] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n07:127355] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n07:127355] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n07:127355] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n09:152061] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n09:152061] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n09:152061] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n09:152061] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n09:152061] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n09:152061] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n09:152061] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n09:152061] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n09:152061] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n09:152061] *** End of error message *** +[0x1000000504d8] +[b10n04:169410] [ 1] [0x3fffc368dff0] +[b10n04:169410] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n04:169410] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n04:169410] [a34n11:147713] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n08:156372] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n08:156372] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n08:156372] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n08:156372] [10] [b09n06:167089] *** Process received signal *** +[b09n06:167089] Signal: Segmentation fault (11) +[b09n06:167089] Signal code: (3) +[b09n06:167089] Failing at address: 0xb170000000000a0 +[b09n06:167089] [ 0] [0x1000000504d8] +[b09n06:167089] [ 1] [0x3ffff1cce130] +[b09n06:167089] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n06:167089] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n06:167089] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n06:167089] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n06:167089] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n06:167089] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n06:167089] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n06:167089] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n06:167089] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n06:167089] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n06:167089] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n06:167089] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n06:167089] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n06:167089] *** End of error message *** +[b07n13:165262] *** Process received signal *** +[b07n13:165262] Signal: Segmentation fault (11) +[b07n13:165262] Signal code: (3) +[b07n13:165262] Failing at address: 0x14170000000000a0 +[b07n13:165262] [ 0] [0x1000000504d8] +[b07n13:165262] [ 1] [0x3fffe7b0b1c0] +[b07n13:165262] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n13:165262] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n06:60151] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n06:60151] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n06:60151] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n06:60151] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n06:60151] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n06:60151] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n06:60151] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n06:60151] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n06:60151] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n18:145742] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n18:145742] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n18:145742] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n18:145742] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n18:145742] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n15:166706] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n15:166706] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n15:166706] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n15:166706] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n15:166706] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n15:166706] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n16:166690] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n16:166690] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n16:166690] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n16:166690] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n16:166690] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n16:166690] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n16:166690] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n16:166690] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n16:166690] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n02:155380] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n02:155380] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n02:155380] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n02:155380] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n02:155380] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n02:155380] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n02:155380] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n02:155380] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n02:155380] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n02:155380] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n02:155380] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n12:115637] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n12:115637] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n12:115637] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n12:115637] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n12:115637] [14] [a30n08:163100] *** Process received signal *** +[a30n08:163100] Signal: Segmentation fault (11) +[a30n08:163100] Signal code: (3) +[a30n08:163100] Failing at address: 0xb170000000000a0 +[a30n08:163100] [ 0] [0x1000000504d8] +[a30n08:163100] [ 1] [0x3fffe6fb4aa0] +[a30n08:163100] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n08:163100] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n06:38119] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n06:38119] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n06:38119] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n06:38119] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n06:38119] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n01:165166] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n01:165166] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n01:165166] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n01:165166] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n01:165166] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n01:165166] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n01:165166] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n01:165166] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n01:165166] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n01:165166] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n01:165166] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n05:169786] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n05:169786] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n05:169786] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n05:169786] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n05:169786] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n05:169786] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n05:169786] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n05:169786] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n05:169786] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n05:169786] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n05:169786] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n05:169786] *** End of error message *** +[ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n06:177847] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n06:177847] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n06:177847] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n06:177847] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n06:177847] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n06:177847] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n06:177847] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n06:177847] [10] [a30n15:154383] *** Process received signal *** +[a30n15:154383] Signal: Segmentation fault (11) +[a30n15:154383] Signal code: (3) +[a30n15:154383] Failing at address: 0xb170000000000a0 +[a30n15:154383] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n05:148378] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n05:148378] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n05:148378] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n12:177120] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n12:177120] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n11:175595] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n11:175595] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n11:175595] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n11:175595] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n08:121690] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n08:121690] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n08:121690] [ 6] [b02n03:39684] *** Process received signal *** +[b02n03:39684] Signal: Segmentation fault (11) +[b02n03:39684] Signal code: (3) +[b02n03:39684] Failing at address: 0x14170000000000a0 +[b02n03:39684] [ 0] [0x1000000504d8] +[b02n03:39684] [ 1] [0x3ffff5077050] +[b02n03:39684] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n03:39684] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n09:79007] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n09:79007] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n09:79007] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n11:120242] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n11:120242] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n11:120242] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n11:120242] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n11:120242] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n12:154341] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n12:154341] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n12:154341] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n12:154341] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n12:154341] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n12:154341] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n12:154341] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n15:137008] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n15:137008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n15:137008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n15:137008] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n02:149227] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n02:149227] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n02:149227] [ 8] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n01:138003] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n01:138003] [ 7] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n06:10224] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n02:49105] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n02:49105] [ 6] [a32n11:152816] *** Process received signal *** +[a32n11:152816] Signal: Segmentation fault (11) +[a32n11:152816] Signal code: (3) +[a32n11:152816] Failing at address: 0xb170000000000a0 +[a32n11:152816] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n11:02684] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n11:02684] [10] [0x1000000504d8] +[a35n17:146794] [ 1] [0x3fffeedc3e40] +[a35n17:146794] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n17:146794] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n17:146794] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n12:140120] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n12:140120] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n12:140120] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n12:140120] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n15:123040] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n15:123040] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n15:123040] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n15:123040] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n15:123040] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n09:164363] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n09:164363] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n09:164363] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n03:168930] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n03:168930] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n03:135164] *** End of error message *** +[b04n12:11419] *** Process received signal *** +[b04n12:11419] Signal: Segmentation fault (11) +[b04n12:11419] Signal code: (3) +[b04n12:11419] Failing at address: 0x14170000000000a0 +[b04n12:11419] [ 0] [0x1000000504d8] +[b04n12:11419] [ 1] [0x3fffe043e0c0] +[b04n12:11419] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n12:11419] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n17:119468] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n17:119468] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n17:119468] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n17:119468] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n17:126526] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n17:126526] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[0x3ffffcbd4e50] +[b06n15:161756] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n15:161756] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n15:161756] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n15:161756] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n01:153682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n01:153682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n01:153682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n01:153682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n01:153682] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n14:21317] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n14:21317] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n14:21317] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n14:21317] [10] [0x1000000504d8] +[b22n16:124793] [ 1] [0x3fffc6487470] +[b22n16:124793] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n16:124793] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n16:124793] [ 4] [a30n03:167776] *** Process received signal *** +[a30n03:167776] Signal: Segmentation fault (11) +[a30n03:167776] Signal code: (3) +[a30n03:167776] Failing at address: 0x5170000000000a0 +[a30n03:167776] [ 0] [0x1000000504d8] +[a30n03:167776] [ 1] [0x3ffff189b930] +[a30n03:167776] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n03:167776] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n03:167776] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n03:167776] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n03:167776] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n03:167776] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n03:167776] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n03:167776] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n03:167776] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n03:167776] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n03:167776] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n03:167776] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n03:167776] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n03:167776] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n06:177003] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n06:177003] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n06:177003] [ 8] [a33n09:147791] *** Process received signal *** +[a33n09:147791] Signal: Segmentation fault (11) +[a33n09:147791] Signal code: (3) +[a33n09:147791] Failing at address: 0xb170000000000a0 +[a33n09:147791] [ 0] [0x1000000504d8] +[a33n09:147791] [ 1] [0x3fffee69da40] +[a33n09:147791] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n09:147791] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n09:147791] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n09:147791] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n09:147791] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n09:147791] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n09:147791] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n09:147791] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n03:136667] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n03:136667] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n03:136667] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n03:136667] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n03:136667] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n01:65726] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n01:65726] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n01:65726] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n01:65726] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n01:65726] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n03:125241] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n03:125241] [ 6] [b11n05:162566] *** Process received signal *** +[b11n05:162566] Signal: Segmentation fault (11) +[b11n05:162566] Signal code: (3) +[b11n05:162566] Failing at address: 0x14170000000000a0 +[b11n05:162566] [ 0] [0x1000000504d8] +[b11n05:162566] [ 1] [0x3fffda4d0890] +[a32n06:154623] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n06:154623] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n06:154623] [ 4] [b06n05:163961] *** Process received signal *** +[b06n05:163961] Signal: Segmentation fault (11) +[b06n05:163961] Signal code: (3) +[b06n05:163961] Failing at address: 0x14170000000000a0 +[b06n05:163961] [ 0] [0x1000000504d8] +[b06n05:163961] [ 1] [0x3fffd09c4020] +[b06n05:163961] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n05:163961] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n05:163961] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n05:163961] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n05:163961] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n05:163961] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n05:163961] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n05:163961] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n05:163961] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n05:163961] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n05:163961] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n05:163961] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n05:163961] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n05:163961] *** End of error message *** +[b01n04:169705] *** Process received signal *** +[b01n04:169705] Signal: Segmentation fault (11) +[b01n04:169705] Signal code: (3) +[b01n04:169705] Failing at address: 0x5170000000000a0 +[b01n04:169705] [ 0] [0x1000000504d8] +[b01n04:169705] [ 1] [0x3fffc896d850] +[b01n04:169705] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n04:169705] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n04:169705] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n04:169705] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n04:169705] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n04:169705] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n04:169705] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n04:169705] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n03:01352] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n04:118345] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n11:148674] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n11:148674] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n11:148674] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n11:148674] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n04:54379] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n04:54379] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n04:54379] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n04:54379] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n04:54379] *** End of error message *** +[b17n02:127219] *** Process received signal *** +[b17n02:127219] Signal: Segmentation fault (11) +[b17n02:127219] Signal code: (3) +[b17n02:127219] Failing at address: 0x14170000000000a0 +[b17n02:127219] [ 0] [0x1000000504d8] +[b17n02:127219] [ 1] [0x3fffdd0b5120] +[b17n02:127219] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n02:127219] [ 3] [a28n02:178474] Signal: Segmentation fault (11) +[a28n02:178474] Signal code: (3) +[a28n02:178474] Failing at address: 0xb170000000000a0 +[a28n02:178474] [ 0] [0x1000000504d8] +[a28n02:178474] [ 1] [0x3ffff8f4b9f0] +[a28n02:178474] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n02:178474] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n02:178474] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n02:123707] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n02:123707] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n02:123707] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n02:123707] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n05:131261] [ 6] [b15n02:122555] *** Process received signal *** +[b15n02:122555] Signal: Segmentation fault (11) +[b15n02:122555] Signal code: (3) +[b15n02:122555] Failing at address: 0xb170000000000a0 +[b15n02:122555] [ 0] [0x1000000504d8] +[b15n02:122555] [ 1] [0x3fffe8e7a500] +[b15n02:122555] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n02:122555] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n17:163816] [ 4] [0x1000000504d8] +[b11n10:161717] [ 1] [0x3fffcbac9420] +[b11n10:161717] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n10:161717] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n10:161717] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n08:172426] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n08:172426] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n08:172426] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n08:172426] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n07:03459] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n07:03459] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n07:03459] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n07:03459] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n06:130543] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n06:130543] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n06:130543] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n06:130543] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n01:164963] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n01:164963] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n01:164963] [14] [b07n17:162357] *** Process received signal *** +[b07n17:162357] Signal: Segmentation fault (11) +[b07n17:162357] Signal code: (3) +[b07n17:162357] Failing at address: 0xb170000000000a0 +[b07n17:162357] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n06:133952] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n06:133952] [ 6] [b08n09:174692] *** Process received signal *** +[b08n09:174692] Signal: Segmentation fault (11) +[b08n09:174692] Signal code: (3) +[b08n09:174692] Failing at address: 0xb170000000000a0 +[b08n09:174692] [ 0] [0x1000000504d8] +[b08n09:174692] [ 1] [0x3fffc026c860] +[b08n09:174692] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n09:174692] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n09:174692] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n09:174692] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n09:174692] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n09:174692] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n09:174692] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n09:174692] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n09:174692] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n09:174692] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n09:174692] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n09:174692] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n09:174692] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n09:174692] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n05:135122] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n05:135122] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n05:135122] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n05:135122] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n05:135122] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n04:13081] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n04:13081] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n04:13081] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n04:13081] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n04:13081] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n01:179262] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n01:179262] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n01:179262] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n01:179262] [ 9] [a32n03:151879] *** Process received signal *** +[a32n03:151879] Signal: Segmentation fault (11) +[a32n03:151879] Signal code: (3) +[a32n03:151879] Failing at address: 0xb170000000000a0 +[a32n03:151879] [ 0] [0x1000000504d8] +[a32n03:151879] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n02:147884] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n02:147884] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n02:147884] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n02:147884] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n08:175805] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n08:175805] [ 5] [b07n09:163514] *** Process received signal *** +[b07n09:163514] Signal: Segmentation fault (11) +[b07n09:163514] Signal code: (3) +[b07n09:163514] Failing at address: 0xb170000000000a0 +[b07n09:163514] [ 0] [0x1000000504d8] +[b07n09:163514] [ 1] [0x3fffc95ce8d0] +[b07n09:163514] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n09:163514] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n09:163514] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n09:163514] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n09:163514] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n09:163514] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n09:163514] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n09:163514] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n09:163514] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n09:163514] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n05:54534] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n05:54534] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n05:54534] [ 6] [b08n11:164993] *** Process received signal *** +[b08n11:164993] Signal: Segmentation fault (11) +[b08n11:164993] Signal code: (3) +[b08n11:164993] Failing at address: 0x5170000000000a0 +[b08n11:164993] [ 0] [0x1000000504d8] +[b08n11:164993] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n04:142256] *** End of error message *** +[b08n01:171786] *** Process received signal *** +[b08n01:171786] Signal: Segmentation fault (11) +[b08n01:171786] Signal code: (3) +[b08n01:171786] Failing at address: 0x14170000000000a0 +[b08n01:171786] [ 0] [0x1000000504d8] +[b08n01:171786] [ 1] [a36n03:153849] Signal: Segmentation fault (11) +[a36n03:153849] Signal code: (3) +[a36n03:153849] Failing at address: 0xb170000000000a0 +[a36n03:153849] [ 0] [0x1000000504d8] +[a36n03:153849] [ 1] [0x3fffe2decd90] +[a36n03:153849] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n17:118574] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n17:118574] [ 5] [b21n05:130804] *** Process received signal *** +[b21n05:130804] Signal: Segmentation fault (11) +[b21n05:130804] Signal code: (3) +[b21n05:130804] Failing at address: 0x14170000000000a0 +[b21n05:130804] [ 0] [0x1000000504d8] +[b21n05:130804] [ 1] [0x3fffcc41e3c0] +[b21n05:130804] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n05:130804] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n05:130804] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n18:118680] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n18:118680] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n18:118680] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n18:118680] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n18:118680] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n05:154185] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n05:154185] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n05:154185] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n05:154185] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n05:154185] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n05:154185] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n05:154185] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n05:154185] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n05:154185] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n01:141340] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n01:141340] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n01:141340] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n01:141340] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n10:164939] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n10:164939] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n10:164939] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n02:121682] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n02:121682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n02:121682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n02:121682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n02:121682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n02:121682] [ 9] [b15n16:121929] *** Process received signal *** +[b15n16:121929] Signal: Segmentation fault (11) +[b15n16:121929] Signal code: (3) +[b15n16:121929] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n06:162920] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n06:162920] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n06:162920] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n06:162920] [13] [a36n14:152037] *** Process received signal *** +[a36n14:152037] Signal: Segmentation fault (11) +[a36n14:152037] Signal code: (3) +[a36n14:152037] Failing at address: 0xb170000000000a0 +[a36n14:152037] [ 0] [0x1000000504d8] +[a36n14:152037] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n01:158306] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n01:158306] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n01:158306] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n01:158306] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n01:158306] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n06:171503] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n06:171503] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n06:171503] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n06:171503] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n04:123054] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n04:123054] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n04:123054] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n04:123054] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n04:123054] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n04:123054] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n04:123054] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n08:164752] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n08:164752] [a36n09:140254] *** Process received signal *** +[a36n09:140254] Signal: Segmentation fault (11) +[a36n09:140254] Signal code: (3) +[a36n09:140254] Failing at address: 0xb170000000000a0 +[a36n09:140254] [ 0] [0x1000000504d8] +[a36n09:140254] [ 1] [0x3ffff24aa9d0] +[a36n09:140254] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n09:140254] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n09:140254] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n09:140254] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n09:140254] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n09:140254] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n09:140254] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n09:140254] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n09:140254] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n09:140254] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n09:140254] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n09:140254] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n09:140254] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n09:140254] *** End of error message *** +[b22n06:133052] *** Process received signal *** +[b22n06:133052] Signal: Segmentation fault (11) +[b22n06:133052] Signal code: (3) +[b22n06:133052] Failing at address: 0xb170000000000a0 +[b22n06:133052] [ 0] [0x1000000504d8] +[b22n06:133052] [ 1] [0x3fffd1019cb0] +[b22n06:133052] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n06:133052] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n06:133052] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n06:133052] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n06:133052] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n06:133052] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n06:133052] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n06:133052] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n06:133052] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n06:133052] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n06:133052] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n06:133052] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n06:133052] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n06:133052] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n01:132062] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n01:132062] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n01:132062] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n03:121810] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n03:121810] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n03:121810] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n03:121810] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n03:121810] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n03:121810] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n03:121810] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n03:121810] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n03:121810] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n03:121810] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n10:02711] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n10:02711] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n10:02711] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n10:02711] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n10:02711] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n10:02711] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n10:02711] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n10:02711] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n04:04850] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n04:04850] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n04:04850] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n04:04850] [10] [b15n04:133903] *** Process received signal *** +[b15n04:133903] Signal: Segmentation fault (11) +[b15n04:133903] Signal code: (3) +[b15n04:133903] Failing at address: 0xb170000000000a0 +[b15n04:133903] [ 0] [0x1000000504d8] +[b15n04:133903] [ 1] [0x3fffcbf1fa80] +[b15n04:133903] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n04:133903] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n04:133903] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n04:133903] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n04:133903] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n04:133903] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n04:133903] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n04:133903] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n04:133903] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n04:133903] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n04:133903] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n04:133903] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n04:133903] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n04:133903] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n02:139024] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n02:139024] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n02:139024] [ 6] [0x3ffff5c121e0] +[b23n02:116888] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n02:116888] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n02:116888] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n02:116888] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n02:116888] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n16:171168] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n16:171168] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n16:171168] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n16:171168] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n16:171168] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n16:171168] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n16:171168] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n16:171168] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n16:171168] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n16:171168] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n05:02909] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n05:02909] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n05:02909] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n05:02909] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n05:02909] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n05:02909] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n05:02909] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n07:119635] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n07:119635] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n07:119635] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n07:119635] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n07:119635] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n06:120735] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n06:120735] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n06:120735] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n07:03371] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n07:03371] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n07:03371] [ 6] [b10n18:162423] *** Process received signal *** +[b10n18:162423] Signal: Segmentation fault (11) +[b10n18:162423] Signal code: (3) +[b10n18:162423] Failing at address: 0xb170000000000a0 +[b10n18:162423] [ 0] [0x1000000504d8] +[b10n18:162423] [ 1] [0x3fffee2ea7b0] +[b10n18:162423] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n18:162423] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n18:162423] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n18:162423] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n18:162423] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n18:162423] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n18:162423] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n18:162423] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n18:162423] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n18:162423] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n18:162423] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n18:162423] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n18:162423] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n18:162423] *** End of error message *** +[b02n15:169491] *** Process received signal *** +[b02n15:169491] Signal: Segmentation fault (11) +[b02n15:169491] Signal code: (3) +[b02n15:169491] Failing at address: 0xb170000000000a0 +[b02n15:169491] [ 0] [0x1000000504d8] +[b02n15:169491] [ 1] [0x3fffec3a61e0] +[b02n15:169491] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n15:169491] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n15:169491] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n15:169491] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n15:169491] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n15:169491] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n15:169491] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n15:169491] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n15:169491] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n15:169491] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n15:169491] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n15:169491] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n15:169491] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n15:169491] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n05:122272] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n05:122272] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n05:122272] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n05:122272] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n05:122272] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n05:122272] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n05:122272] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n08:121433] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n08:121433] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n08:121433] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n08:121433] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n08:121433] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n08:121433] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n03:130346] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n03:130346] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n03:130346] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n03:130346] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n03:130346] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n03:130346] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n06:132107] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n06:132107] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n06:132107] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n06:132107] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n06:132107] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n06:132107] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n06:163678] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n06:163678] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n06:163678] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n06:163678] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n06:163678] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n06:163678] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n05:173080] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n05:173080] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n11:160799] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n11:160799] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n11:160799] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n01:128005] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n01:128005] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n01:128005] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n05:120332] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n05:120332] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n05:120332] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n16:164549] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n16:164549] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n16:164549] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n12:138738] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n12:138738] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n12:138738] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n12:138738] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n04:126861] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n04:126861] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n04:126861] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n04:126861] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n04:126861] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n04:126861] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n04:126861] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n04:126861] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n04:126861] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n04:126861] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n05:126782] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n05:126782] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n05:126782] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n05:126782] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n05:126782] [14] [a32n15:139055] *** Process received signal *** +[a32n15:139055] Signal: Segmentation fault (11) +[a32n15:139055] Signal code: (3) +[a32n15:139055] Failing at address: 0xb170000000000a0 +[a32n15:139055] [ 0] [0x1000000504d8] +[a32n15:139055] [ 1] [0x3fffe2732430] +[a32n15:139055] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n15:139055] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n15:139055] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n15:139055] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n15:139055] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n15:139055] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n15:139055] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n15:139055] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n15:139055] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n15:139055] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n15:139055] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n15:139055] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n15:139055] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n15:139055] *** End of error message *** +[b19n17:123230] *** Process received signal *** +[b19n17:123230] Signal: Segmentation fault (11) +[b19n17:123230] Signal code: (3) +[b19n17:123230] Failing at address: 0xb170000000000a0 +[b19n17:123230] [ 0] [0x1000000504d8] +[b19n17:123230] [ 1] [0x3ffffbe8f7f0] +[b19n17:123230] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n17:123230] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n07:124352] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n07:124352] [10] [b05n13:10195] *** Process received signal *** +[b05n13:10195] Signal: Segmentation fault (11) +[b05n13:10195] Signal code: (3) +[b05n13:10195] Failing at address: 0xb170000000000a0 +[b05n13:10195] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n09:155757] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n09:155757] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n09:155757] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n17:176414] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n17:176414] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n17:176414] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n17:176414] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n01:165124] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n01:165124] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n01:165124] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n01:165124] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n01:165124] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n01:165124] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n01:165124] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n01:165124] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n01:165124] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n06:157789] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n11:117475] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n11:117475] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n11:117475] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n06:161721] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n06:138765] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n06:138765] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n06:138765] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n06:138765] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n06:138765] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n06:138765] *** End of error message *** +[b01n12:178599] *** Process received signal *** +[b01n12:178599] Signal: Segmentation fault (11) +[b01n12:178599] Signal code: (3) +[b01n12:178599] Failing at address: 0x14170000000000a0 +[b01n12:178599] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n06:117526] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n06:117526] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n06:117526] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n06:117526] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n06:117526] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n07:152084] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n07:152084] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n07:152084] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n07:152084] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n07:152084] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n07:152084] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n07:152084] [10] [a33n17:168155] *** Process received signal *** +[a33n17:168155] Signal: Segmentation fault (11) +[a33n17:168155] Signal code: (3) +[a33n17:168155] Failing at address: 0xb170000000000a0 +[a33n17:168155] [ 0] [0x1000000504d8] +[a33n17:168155] [ 1] [b11n13:153677] *** Process received signal *** +[b11n13:153677] Signal: Segmentation fault (11) +[b11n13:153677] Signal code: (3) +[b11n13:153677] Failing at address: 0xb170000000000a0 +[b11n13:153677] [ 0] [0x1000000504d8] +[b11n13:153677] [ 1] [0x3fffc35e4b00] +[b11n13:153677] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n13:153677] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n10:16972] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n05:139434] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n05:139434] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n05:139434] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n05:139434] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n03:120668] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n03:120668] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n03:120668] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n03:120668] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n07:160802] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n07:160802] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n07:160802] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n07:160802] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n11:120580] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n11:120580] [14] [a27n03:46691] *** Process received signal *** +[a27n03:46691] Signal: Segmentation fault (11) +[a27n03:46691] Signal code: (3) +[a27n03:46691] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n04:158636] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n04:158636] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n04:158636] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n04:158636] [14] [b08n14:178326] *** Process received signal *** +[b08n14:178326] Signal: Segmentation fault (11) +[b08n14:178326] Signal code: (3) +[b08n14:178326] Failing at address: 0x14170000000000a0 +[b08n14:178326] [ 0] [0x1000000504d8] +[b08n14:178326] [ 1] [0x3fffc7bb60d0] +[b08n14:178326] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n14:178326] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n09:119635] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n09:119635] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n09:119635] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n09:119635] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n03:154296] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n03:154296] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n03:154296] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n03:154296] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n03:154296] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n03:154296] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n04:145749] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n04:145749] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n13:171109] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n13:171109] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n13:171109] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n13:171109] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n11:03984] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n11:03984] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n11:03984] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n11:03984] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n11:03984] [ 7] [b22n05:123933] Signal: Segmentation fault (11) +[b22n05:123933] Signal code: (3) +[b22n05:123933] Failing at address: 0xb170000000000a0 +[b22n05:123933] [ 0] [0x1000000504d8] +[b22n05:123933] [ 1] [0x3fffebaf04c0] +[b22n05:123933] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n05:123933] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n05:123933] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n05:123933] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n05:123933] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n05:123933] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n05:123933] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n05:123933] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n05:123933] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n05:123933] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n05:123933] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n05:123933] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n05:123933] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n05:123933] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n10:11489] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n10:11489] *** End of error message *** +[0x1000000504d8] +[a29n17:169734] [ 1] [0x3fffe44abc60] +[a29n17:169734] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n17:169734] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n17:169734] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n07:168963] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n07:168963] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n07:168963] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n07:168963] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n07:168963] *** End of error message *** +[a28n18:179169] *** Process received signal *** +[a28n18:179169] Signal: Segmentation fault (11) +[a28n18:179169] Signal code: (3) +[a28n18:179169] Failing at address: 0x14170000000000a0 +[a28n18:179169] [ 0] [0x1000000504d8] +[a28n18:179169] [ 1] [0x3fffd6429640] +[a28n18:179169] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n18:179169] [ 3] [0x1000000504d8] +[b13n11:134518] [ 1] [0x3fffebc37fb0] +[b13n11:134518] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n11:134518] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n11:134518] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n04:44756] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n04:44756] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n13:123517] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n13:123517] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n13:123517] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n13:123517] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n12:123558] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n16:53798] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n16:53798] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n16:53798] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n16:53798] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n16:53798] *** End of error message *** +[a36n05:139912] *** Process received signal *** +[a36n05:139912] Signal: Segmentation fault (11) +[a36n05:139912] Signal code: (3) +[a36n05:139912] Failing at address: 0x5170000000000a0 +[a30n05:156007] *** Process received signal *** +[a30n05:156007] Signal: Segmentation fault (11) +[a30n05:156007] Signal code: (3) +[a30n05:156007] Failing at address: 0x14170000000000a0 +[a30n05:156007] [ 0] [0x1000000504d8] +[a30n05:156007] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n10:169211] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n10:169211] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n10:169211] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n10:169211] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n10:169211] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n10:169211] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n13:123055] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n13:123055] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n13:123055] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n13:123055] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n14:08398] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n05:174782] *** End of error message *** +[a30n13:154231] *** Process received signal *** +[a30n13:154231] Signal: Segmentation fault (11) +[a30n13:154231] Signal code: (3) +[a30n13:154231] Failing at address: 0xb170000000000a0 +[a30n13:154231] [ 0] [0x1000000504d8] +[a30n13:154231] [ 1] [0x3ffff38389a0] +[a30n13:154231] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n13:154231] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n14:133965] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n14:133965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n14:133965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n14:133965] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n14:133965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n14:133965] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n14:133965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n14:133965] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n14:133965] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n14:133965] *** End of error message *** +[a30n04:03148] *** Process received signal *** +[a30n04:03148] Signal: Segmentation fault (11) +[a30n04:03148] Signal code: (3) +[a30n04:03148] Failing at address: 0x14170000000000a0 +[a30n04:03148] [ 0] [0x1000000504d8] +[a30n04:03148] [ 1] [0x3fffecb70830] +[a30n04:03148] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n04:03148] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n08:141874] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n08:141874] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n08:176680] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n08:176680] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n08:176680] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n08:176680] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n08:176680] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n08:176680] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n01:163391] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n01:163391] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n01:163391] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n01:163391] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n03:06825] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n03:06825] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n03:06825] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n10:118386] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n10:118386] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n10:118386] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n12:160964] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n12:160964] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n12:160964] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n12:160964] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n12:160964] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n16:162628] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n16:162628] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n16:162628] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n16:162628] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n16:162628] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n16:162628] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n16:162628] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n16:162628] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n16:162628] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n16:162628] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n16:162628] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n16:162628] *** End of error message *** +[0x3fffd9486bf0] +[b04n01:96754] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n01:96754] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n01:96754] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n15:166119] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n15:166119] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n15:166119] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n15:166119] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n15:166119] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n15:166119] [14] [b04n09:12246] *** Process received signal *** +[b04n09:12246] Signal: Segmentation fault (11) +[b04n09:12246] Signal code: (3) +[b04n09:12246] Failing at address: 0x14170000000000a0 +[b04n09:12246] [ 0] [0x1000000504d8] +[b04n09:12246] [ 1] [0x1000000504d8] +[a34n12:145497] [ 1] [0x3fffe264dcf0] +[a34n12:145497] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n12:145497] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n12:145497] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n12:145497] [ 5] [b14n11:130803] *** Process received signal *** +[b14n11:130803] Signal: Segmentation fault (11) +[b14n11:130803] Signal code: (3) +[b14n11:130803] Failing at address: 0x14170000000000a0 +[b14n11:130803] [ 0] [0x1000000504d8] +[b14n11:130803] [ 1] [0x3fffc08217c0] +[b14n11:130803] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n11:130803] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n18:123412] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n18:123412] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n18:123412] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n18:123412] *** End of error message *** +[a28n06:179453] *** Process received signal *** +[a28n06:179453] Signal: Segmentation fault (11) +[a28n06:179453] Signal code: (3) +[a28n06:179453] Failing at address: 0xb170000000000a0 +[a28n06:179453] [ 0] [0x1000000504d8] +[a28n06:179453] [ 1] [0x3fffc28bf520] +[a28n06:179453] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n06:179453] [ 3] [b03n01:170767] *** Process received signal *** +[b03n01:170767] Signal: Segmentation fault (11) +[b03n01:170767] Signal code: (3) +[b03n01:170767] Failing at address: 0x14170000000000a0 +[b03n01:170767] [ 0] [0x1000000504d8] +[b03n01:170767] [ 1] [0x3ffffb928350] +[b03n01:170767] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n01:170767] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n18:128539] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n18:128539] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n18:128539] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n18:128539] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n18:128539] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n18:128539] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n18:128539] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n05:120230] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n05:120230] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n08:159026] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n08:159026] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n08:159026] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n08:159026] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n08:159026] [10] [0x1000000504d8] +[a33n10:140416] [ 1] [0x3fffe40628b0] +[a33n10:140416] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n10:140416] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n10:140416] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n10:140416] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n10:140416] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n10:140416] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n10:140416] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n03:172648] *** End of error message *** +[b11n16:154183] *** Process received signal *** +[b11n16:154183] Signal: Segmentation fault (11) +[b11n16:154183] Signal code: (3) +[b11n16:154183] Failing at address: 0xb170000000000a0 +[b11n16:154183] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n15:120132] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n15:120132] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n15:120132] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n15:120132] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n13:133624] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n13:133624] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n13:133624] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n13:133624] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n06:119114] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n06:119114] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n06:119114] [10] [b17n10:126612] *** Process received signal *** +[b17n10:126612] Signal: Segmentation fault (11) +[b17n10:126612] Signal code: (3) +[b17n10:126612] Failing at address: 0xb170000000000a0 +[b17n10:126612] [ 0] [0x1000000504d8] +[b17n10:126612] [ 1] [0x3fffed379e10] +[b17n10:126612] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n10:126612] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n10:126612] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n10:126612] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n01:124278] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n01:124278] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n11:156154] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n11:156154] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n11:156154] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n11:156154] [ 9] [0x3fffe1d1ddc0] +[b04n08:176174] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n08:176174] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n08:176174] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n08:176174] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n08:176174] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n08:176174] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n08:176174] [ 8] [b20n02:122181] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n05:04521] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n05:04521] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n05:04521] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n05:04521] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n10:10922] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n10:10922] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n10:10922] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n10:10922] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n10:10922] *** End of error message *** +[b15n15:122085] *** Process received signal *** +[b15n15:122085] Signal: Segmentation fault (11) +[b15n15:122085] Signal code: (3) +[b15n15:122085] Failing at address: 0xb170000000000a0 +[b15n15:122085] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n15:139746] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n15:139746] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n15:139746] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n15:139746] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n08:127445] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n08:127445] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n08:127445] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n08:127445] [ 9] [a30n18:10724] Signal: Segmentation fault (11) +[a30n18:10724] Signal code: (3) +[a30n18:10724] Failing at address: 0x14170000000000a0 +[a30n18:10724] [ 0] [0x1000000504d8] +[a30n18:10724] [ 1] [0x3fffc8d181e0] +[a30n18:10724] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n18:10724] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n09:168245] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n09:168245] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n09:168245] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n09:168245] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n09:168245] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n09:168245] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n14:126464] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n14:126464] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n14:126464] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n14:126464] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n14:126464] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n14:126464] *** End of error message *** +[b09n13:164687] *** Process received signal *** +[b09n13:164687] Signal: Segmentation fault (11) +[b09n13:164687] Signal code: (3) +[b09n13:164687] Failing at address: 0x14170000000000a0 +[b09n13:164687] [ 0] [0x1000000504d8] +[b09n13:164687] [ 1] [0x3ffff202c620] +[b09n13:164687] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n13:164687] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n13:164687] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n13:164687] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n13:164687] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n13:164687] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n13:164687] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n13:164687] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n13:164687] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n13:164687] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n13:164687] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n13:164687] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n13:164687] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n13:164687] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n07:165091] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n07:165091] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n03:09642] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n17:138616] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n17:138616] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n17:138616] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n17:138616] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n17:138616] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n17:138616] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n17:138616] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n17:138616] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n05:173316] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n05:173316] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n05:173316] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n05:173316] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n05:173316] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n05:173316] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n05:173316] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n05:173316] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n05:173316] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n05:173316] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n09:160372] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n09:160372] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n09:160372] [ 6] [a32n14:152747] Signal: Segmentation fault (11) +[a32n14:152747] Signal code: (3) +[a32n14:152747] Failing at address: 0x14170000000000a0 +[a32n14:152747] [ 0] [0x1000000504d8] +[a32n14:152747] [ 1] [0x3fffc2339990] +[a32n14:152747] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n14:152747] [ 3] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n10:169685] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n10:169685] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n10:169685] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n10:155396] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n10:155396] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n10:155396] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n10:155396] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n10:155396] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n01:121743] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n01:121743] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n01:121743] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n01:121743] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n01:121743] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n01:121743] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n07:172876] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n07:172876] *** End of error message *** +[b19n10:120977] *** Process received signal *** +[b19n10:120977] Signal: Segmentation fault (11) +[b19n10:120977] Signal code: (3) +[b19n10:120977] Failing at address: 0x14170000000000a0 +[b19n10:120977] [ 0] [0x1000000504d8] +[b19n10:120977] [ 1] [0x3ffff20d7110] +[b19n10:120977] [ 2] [b05n11:176579] *** Process received signal *** +[b05n11:176579] Signal: Segmentation fault (11) +[b05n11:176579] Signal code: (3) +[b05n11:176579] Failing at address: 0x14170000000000a0 +[b05n11:176579] [ 0] [0x1000000504d8] +[b05n11:176579] [ 1] [0x3fffca748600] +[b05n11:176579] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n11:176579] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n11:176579] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n11:176579] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n11:176579] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n11:176579] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n11:176579] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n11:176579] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n11:176579] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n11:176579] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n11:176579] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n11:176579] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n11:176579] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n11:176579] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n02:74428] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n02:74428] *** End of error message *** +[a31n18:169358] *** Process received signal *** +[a31n18:169358] Signal: Segmentation fault (11) +[a31n18:169358] Signal code: (3) +[a31n18:169358] Failing at address: 0xb170000000000a0 +[a31n18:169358] [ 0] [0x1000000504d8] +[a31n18:169358] [ 1] [0x3fffdce37990] +[a31n18:169358] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n18:169358] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n18:169358] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n18:169358] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n18:169358] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n18:169358] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n18:169358] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n18:169358] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n18:169358] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n18:169358] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n18:169358] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n18:169358] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n18:169358] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n18:169358] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n10:135852] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n10:135852] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n10:135852] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n10:135852] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n10:135852] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n02:125266] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n02:125266] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n02:125266] [14] [a31n02:158416] *** Process received signal *** +[a31n02:158416] Signal: Segmentation fault (11) +[a31n02:158416] Signal code: (3) +[a31n02:158416] Failing at address: 0xb170000000000a0 +[a31n02:158416] [ 0] [0x1000000504d8] +[a31n02:158416] [ 1] [0x3ffffb4d8060] +[a31n02:158416] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n02:158416] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n02:158416] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n02:158416] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n02:158416] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n02:158416] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n02:158416] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n02:158416] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n02:158416] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n02:158416] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n02:158416] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n02:158416] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n02:158416] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n02:158416] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n02:125370] *** End of error message *** +[b06n03:164495] *** Process received signal *** +[b19n12:122875] *** Process received signal *** +[b19n12:122875] Signal: Segmentation fault (11) +[b19n12:122875] Signal code: (3) +[b19n12:122875] Failing at address: 0x14170000000000a0 +[b19n12:122875] [ 0] [b19n04:130747] *** Process received signal *** +[b19n04:130747] Signal: Segmentation fault (11) +[b19n04:130747] Signal code: (3) +[b19n04:130747] Failing at address: 0x14170000000000a0 +[b19n04:130747] [ 0] [0x1000000504d8] +[b19n04:130747] [ 1] [0x1000000504d8] +[b18n16:124960] [ 1] [0x3fffd94d7b90] +[b18n16:124960] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n16:124960] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n16:124960] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n16:124960] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n07:45305] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n07:45305] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n07:45305] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n07:45305] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n07:45305] *** End of error message *** +[b07n02:174054] *** Process received signal *** +[b07n02:174054] Signal: Segmentation fault (11) +[b07n02:174054] Signal code: (3) +[b07n02:174054] Failing at address: 0xb170000000000a0 +[b07n02:174054] [ 0] [0x1000000504d8] +[b07n02:174054] [ 1] [0x3fffe27540f0] +[b07n02:174054] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n02:174054] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n02:174054] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n02:174054] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n02:174054] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n02:174054] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n02:174054] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n02:174054] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n02:174054] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n02:174054] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n02:174054] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n02:174054] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n02:174054] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n02:174054] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n05:149466] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n05:149466] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n05:149466] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n05:149466] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n05:149466] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n13:155851] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n13:155851] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n13:155851] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n13:155851] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n13:155851] *** End of error message *** +[a29n12:160190] *** Process received signal *** +[a29n12:160190] Signal: Segmentation fault (11) +[a29n12:160190] Signal code: (3) +[a29n12:160190] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n09:106786] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n09:106786] [ 6] [b05n14:177512] *** Process received signal *** +[b05n14:177512] Signal: Segmentation fault (11) +[b05n14:177512] Signal code: (3) +[b05n14:177512] Failing at address: 0x14170000000000a0 +[b05n14:177512] [ 0] [0x1000000504d8] +[b05n14:177512] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n09:170658] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n09:170658] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n09:170658] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n09:170658] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n09:170658] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n09:170658] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n09:170658] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n17:122355] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n17:122355] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n17:122355] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n17:122355] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n17:122355] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n05:176911] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n05:176911] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n10:122756] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n10:122756] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n10:122756] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n10:122756] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n08:146119] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n08:146119] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n04:156414] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n04:156414] [ 8] [b17n03:135037] *** Process received signal *** +[b17n03:135037] Signal: Segmentation fault (11) +[b17n03:135037] Signal code: (3) +[b17n03:135037] Failing at address: 0x5170000000000a0 +[b17n03:135037] [ 0] [0x1000000504d8] +[b17n03:135037] [ 1] [0x3ffff54b2a40] +[b17n03:135037] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n03:135037] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n07:156487] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n07:156487] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n07:156487] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n07:156487] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n07:156487] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n07:156487] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n07:156487] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n07:156487] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n07:156487] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n07:156487] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n07:156487] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n07:156487] *** End of error message *** +[0x3fffee2630c0] +[b09n17:163177] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n17:163177] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n17:163177] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n09:128493] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n09:128493] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n09:128493] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n07:02840] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n07:02840] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n14:126119] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n14:126119] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n14:126119] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n08:118554] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n08:118554] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n08:118554] *** End of error message *** +[b06n10:176263] *** Process received signal *** +[b06n10:176263] Signal: Segmentation fault (11) +[b06n10:176263] Signal code: (3) +[b06n10:176263] Failing at address: 0xb170000000000a0 +[b06n10:176263] [ 0] [0x3ffff42393c0] +[b18n04:136996] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n04:136996] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n04:136996] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n04:136996] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n04:136996] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n04:136996] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n04:136996] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n04:136996] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n04:136996] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n03:126187] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n03:126187] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n03:126187] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n15:165473] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n15:165473] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n15:165473] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n15:165473] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n03:147536] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n03:68832] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n03:68832] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n03:68832] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n03:68832] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n03:68832] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n03:68832] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n11:134749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n11:134749] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n11:134749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n11:134749] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n12:145263] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n08:165281] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n08:165281] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n08:165281] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n08:165281] [13] [a34n13:137748] *** Process received signal *** +[a34n13:137748] Signal: Segmentation fault (11) +[a34n13:137748] Signal code: (3) +[a34n13:137748] Failing at address: 0x14170000000000a0 +[a34n13:137748] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n14:122443] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n14:122443] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n14:122443] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n14:122443] [13] [b15n01:123570] *** Process received signal *** +[b15n01:123570] Signal: Segmentation fault (11) +[b15n01:123570] Signal code: (3) +[b15n01:123570] Failing at address: 0x14170000000000a0 +[b15n01:123570] [ 0] [0x1000000504d8] +[b15n01:123570] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n12:120401] *** End of error message *** +[b15n03:119831] *** Process received signal *** +[b15n03:119831] Signal: Segmentation fault (11) +[b15n03:119831] Signal code: (3) +[b15n03:119831] Failing at address: 0x14170000000000a0 +[b15n03:119831] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n07:125451] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n07:125451] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n07:125451] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n05:173822] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n05:173822] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n05:173822] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n11:128953] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n11:128953] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n11:128953] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n11:128953] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n13:165388] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n13:165388] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n13:165388] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n13:165388] [10] [a34n06:148310] *** End of error message *** +[b17n04:136609] *** Process received signal *** +[b17n04:136609] Signal: Segmentation fault (11) +[b17n04:136609] Signal code: (3) +[b17n04:136609] Failing at address: 0xb170000000000a0 +[b17n04:136609] [ 0] [0x1000000504d8] +[b17n04:136609] [ 1] [0x3fffd72201c0] +[b17n04:136609] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n04:136609] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n03:155729] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n03:124681] [ 4] [b07n07:163254] *** Process received signal *** +[b07n07:163254] Signal: Segmentation fault (11) +[b07n07:163254] Signal code: (3) +[b07n07:163254] Failing at address: 0x14170000000000a0 +[b07n07:163254] [ 0] [0x1000000504d8] +[b07n07:163254] [ 1] [0x3fffc84e1df0] +[b07n07:163254] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n07:163254] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n16:129236] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n08:124119] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n08:124119] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n08:124119] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n08:124119] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n08:124119] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n08:124119] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n12:177496] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n12:177496] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n12:177496] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n12:177496] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n12:177496] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n12:177496] *** End of error message *** +[0x1000000504d8] +[a33n12:168641] [ 1] [0x3fffe84d86c0] +[a33n12:168641] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n12:168641] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n12:168641] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n14:177225] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n14:116410] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n14:116410] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n14:116410] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n14:116410] [10] [b18n18:121307] *** Process received signal *** +[b01n02:01427] *** Process received signal *** +[b01n02:01427] Signal: Segmentation fault (11) +[b01n02:01427] Signal code: (3) +[b01n02:01427] Failing at address: 0x14170000000000a0 +[b01n02:01427] [ 0] [0x1000000504d8] +[b01n02:01427] [ 1] [0x3fffd51be280] +[b01n02:01427] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n02:01427] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n02:01427] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n02:01427] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n02:01427] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n02:01427] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n02:01427] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n02:01427] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n02:01427] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n02:01427] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n02:01427] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n02:01427] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n02:01427] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n02:01427] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n05:05607] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n05:05607] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n04:177598] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n04:177598] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n04:177598] [ 6] [b13n09:143896] *** Process received signal *** +[b13n09:143896] Signal: Segmentation fault (11) +[b13n09:143896] Signal code: (3) +[b13n09:143896] Failing at address: 0xb170000000000a0 +[b13n09:143896] [ 0] [0x1000000504d8] +[b13n09:143896] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n04:117355] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n04:117355] *** End of error message *** +[b02n12:180114] *** Process received signal *** +[b02n12:180114] Signal: Segmentation fault (11) +[b02n12:180114] Signal code: (3) +[b02n12:180114] Failing at address: 0xb170000000000a0 +[b02n12:180114] [ 0] [b12n14:139843] *** Process received signal *** +[b12n14:139843] Signal: Segmentation fault (11) +[b12n14:139843] Signal code: (3) +[b12n14:139843] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n13:177637] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n13:177637] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n13:177637] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n13:177637] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n05:53373] [ 5] [0x3ffffb385500] +[b02n09:166574] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n09:166574] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n09:166574] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n09:166574] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n09:166574] [ 6] [0x3fffce265ab0] +[b13n08:144855] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n08:144855] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n08:144855] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n08:144855] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n13:54242] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n08:138174] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n08:138174] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n08:138174] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n08:138174] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n12:119806] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n12:119806] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n12:119806] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n12:119806] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n05:121826] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n05:121826] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n18:03312] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n18:03312] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n18:03312] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n18:03312] [10] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n16:147287] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n16:147287] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n16:147287] [ 5] [a33n07:149476] *** Process received signal *** +[a33n07:149476] Signal: Segmentation fault (11) +[a33n07:149476] Signal code: (3) +[a33n07:149476] Failing at address: 0xb170000000000a0 +[a33n07:149476] [ 0] [0x1000000504d8] +[a33n07:149476] [ 1] [0x3fffea85a920] +[a33n07:149476] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n07:149476] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n04:168491] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n04:168491] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n04:168491] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n07:156140] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n07:156140] [ 6] [0x3fffd0ec6680] +[b09n07:173044] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n07:173044] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n07:173044] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n07:173044] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n07:173044] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n07:173044] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n07:173044] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n07:173044] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n07:173044] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n07:173044] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n07:173044] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n07:173044] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n07:173044] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n07:173044] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n18:122553] [ 6] [b19n16:130377] *** Process received signal *** +[b19n16:130377] Signal: Segmentation fault (11) +[b19n16:130377] Signal code: (3) +[b19n16:130377] Failing at address: 0x14170000000000a0 +[b19n16:130377] [ 0] [0x1000000504d8] +[b19n16:130377] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n17:125808] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n17:125808] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n17:125808] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n17:125808] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n10:126194] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n10:126194] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n10:126194] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n10:126194] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n14:120638] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n14:120638] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n13:138090] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n13:138090] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n13:138090] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n13:138090] [ 8] [a27n15:32401] [12] [a33n14:136136] *** Process received signal *** +[a33n14:136136] Signal: Segmentation fault (11) +[a33n14:136136] Signal code: (3) +[a33n14:136136] Failing at address: 0x14170000000000a0 +[a33n14:136136] [ 0] [0x1000000504d8] +[a33n14:136136] [ 1] [0x3fffcc274230] +[a33n14:136136] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n14:136136] [ 3] [b14n10:124918] *** Process received signal *** +[b14n10:124918] Signal: Segmentation fault (11) +[b14n10:124918] Signal code: (3) +[b14n10:124918] Failing at address: 0xb170000000000a0 +[b14n10:124918] [ 0] [0x1000000504d8] +[b14n10:124918] [ 1] [0x3ffff2245250] +[b14n10:124918] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n10:124918] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n10:124918] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n10:124918] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n10:124918] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n11:124446] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n11:124446] [ 4] [a32n12:164382] *** Process received signal *** +[a32n12:164382] Signal: Segmentation fault (11) +[a32n12:164382] Signal code: (3) +[a32n12:164382] Failing at address: 0x14170000000000a0 +[a32n12:164382] [ 0] [0x1000000504d8] +[a32n12:164382] [ 1] [0x3fffd7e6e430] +[a32n12:164382] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n12:164382] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n15:120057] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n15:120057] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n15:120057] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n10:141234] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n10:141234] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n10:141234] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n10:141234] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n10:141234] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n15:148167] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n15:148167] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n15:148167] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n01:163026] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n01:163026] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n15:151252] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n15:151252] *** End of error message *** +[b21n15:119254] *** Process received signal *** +[b21n15:119254] Signal: Segmentation fault (11) +[b21n15:119254] Signal code: (3) +[b21n15:119254] Failing at address: 0x14170000000000a0 +[b21n15:119254] [ 0] [0x1000000504d8] +[b21n15:119254] [ 1] [0x3fffd0b7f550] +[b21n15:119254] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n15:119254] [ 3] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n09:173060] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n09:173060] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n14:119488] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n14:119488] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n14:119488] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n14:119488] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n14:150133] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n14:150133] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n14:150133] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n14:150133] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n14:150133] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n10:126792] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n10:126792] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n10:126792] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n10:126792] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n10:126792] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n10:126792] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n10:126792] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n10:126792] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n10:126792] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n10:126792] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n18:176809] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n18:176809] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n18:176809] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n18:176809] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n13:117511] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n13:117511] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n13:117511] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n13:117511] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n02:177548] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n02:177548] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n13:166925] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n02:119534] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n02:119534] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n02:119534] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n02:119534] [10] [a30n11:163260] *** Process received signal *** +[a30n11:163260] Signal: Segmentation fault (11) +[a30n11:163260] Signal code: (3) +[a30n11:163260] Failing at address: 0x14170000000000a0 +[a30n11:163260] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n17:168160] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n17:168160] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n17:168160] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n17:168160] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n11:124435] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n11:124435] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n06:164013] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n06:164013] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n06:164013] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n06:164013] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n06:164013] [13] [b13n10:134733] *** Process received signal *** +[b13n10:134733] Signal: Segmentation fault (11) +[b13n10:134733] Signal code: (3) +[b13n10:134733] Failing at address: 0xb170000000000a0 +[b13n10:134733] [ 0] [0x1000000504d8] +[b13n10:134733] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n12:178717] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n04:126345] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n04:126345] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n04:126345] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n04:126345] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n04:126345] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n04:126345] [ 9] [a29n11:162469] *** Process received signal *** +[a29n11:162469] Signal: Segmentation fault (11) +[a29n11:162469] Signal code: (3) +[a29n11:162469] Failing at address: 0xb170000000000a0 +[b09n16:173550] [ 0] [0x1000000504d8] +[b09n16:173550] [ 1] [0x3fffe84edaa0] +[b09n16:173550] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n16:173550] [ 3] [b17n05:125612] *** Process received signal *** +[b17n05:125612] Signal: Segmentation fault (11) +[b17n05:125612] Signal code: (3) +[b17n05:125612] Failing at address: 0x14170000000000a0 +[b17n05:125612] [ 0] [0x1000000504d8] +[b17n05:125612] [ 1] [0x3ffff7ed7130] +[b17n05:125612] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n05:125612] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n11:153415] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n11:153415] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n03:158655] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n03:158655] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n03:158655] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n03:158655] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n18:164797] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n18:164797] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n18:164797] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n18:164797] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n13:168162] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n13:168162] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n13:168162] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n13:168162] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n13:168162] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n04:179173] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n04:179173] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n18:08427] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n18:08427] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n18:08427] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n18:08427] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n18:08427] [10] [a32n04:153962] *** Process received signal *** +[a32n04:153962] Signal: Segmentation fault (11) +[a32n04:153962] Signal code: (3) +[a32n04:153962] Failing at address: 0xb170000000000a0 +[a32n04:153962] [ 0] [0x1000000504d8] +[a32n04:153962] [ 1] [0x3ffffdd2ff80] +[a32n04:153962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n04:153962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n16:149063] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n16:149063] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n16:149063] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n16:149063] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n10:173681] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n10:173681] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n10:173681] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n10:173681] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n10:173681] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n10:173681] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n10:173681] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n10:173681] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n17:129932] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n17:129932] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n17:129932] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n17:129932] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n12:120131] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n12:120131] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n12:120131] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n12:120131] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n12:120131] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n12:120131] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n12:120131] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n09:161232] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n09:161232] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n09:161232] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n09:161232] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n09:161232] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n15:124800] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n15:124800] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n15:124800] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n15:124800] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n15:124800] [10] [0x1000000504d8] +[b19n13:123885] [ 1] [0x3fffc7a065b0] +[b19n13:123885] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n13:123885] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n13:123885] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n18:140382] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n18:140382] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n18:140382] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n18:131584] [ 5] [b02n05:164944] *** Process received signal *** +[b02n05:164944] Signal: Segmentation fault (11) +[b02n05:164944] Signal code: (3) +[b02n05:164944] Failing at address: 0x14170000000000a0 +[b02n05:164944] [ 0] [0x1000000504d8] +[b02n05:164944] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n08:54627] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n08:54627] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n06:127275] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n06:127275] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n06:127275] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n10:162611] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n10:162611] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n10:162611] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n10:162611] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n10:162611] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n10:162611] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n10:162611] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n14:133062] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n14:133062] *** End of error message *** +[a02n04:41672] *** Process received signal *** +[a02n04:41672] Signal: Segmentation fault (11) +[a02n04:41672] Signal code: (3) +[a02n04:41672] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n16:128439] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n16:128439] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n02:167598] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n02:167598] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n16:127353] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n16:127353] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n16:127353] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n16:127353] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n16:127353] [11] [a02n18:21768] *** Process received signal *** +[a02n18:21768] Signal: Segmentation fault (11) +[a02n18:21768] Signal code: (3) +[a02n18:21768] Failing at address: 0x14170000000000a0 +[a02n18:21768] [ 0] [0x1000000504d8] +[a02n18:21768] [ 1] [0x3fffdd1af680] +[a02n18:21768] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n18:21768] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n18:21768] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n18:21768] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n18:21768] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n18:21768] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n18:21768] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n18:21768] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n18:21768] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n18:21768] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n18:21768] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n18:21768] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n18:21768] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n18:21768] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n04:166206] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n04:166206] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n04:166206] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n04:166206] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n04:166206] *** End of error message *** +[b18n08:131380] *** Process received signal *** +[b18n08:131380] Signal: Segmentation fault (11) +[b18n08:131380] Signal code: (3) +[b18n08:131380] Failing at address: 0x14170000000000a0 +[b18n08:131380] [ 0] [0x1000000504d8] +[b18n08:131380] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n08:116172] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n08:116172] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n04:52706] *** End of error message *** +[12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n07:117359] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n14:176070] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n14:176070] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n14:176070] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n14:176070] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n16:165920] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n16:165920] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n16:165920] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n16:165920] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n17:124167] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n02:176050] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n02:176050] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n02:176050] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n02:176050] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n17:171199] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n17:171199] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n17:171199] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n18:07421] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n18:07421] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n18:07421] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n03:123614] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n03:123614] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n03:123614] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n12:132100] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n12:132100] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n12:132100] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n12:132100] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n06:167367] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n06:167367] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n06:167367] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n06:167367] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n06:167367] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n11:23061] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n18:69156] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n14:168375] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n08:172900] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n08:172900] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n08:172900] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n07:133373] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n07:133373] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n07:133373] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n07:133373] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n13:122032] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n13:122032] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n09:149100] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n09:149100] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n09:149100] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n09:149100] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n09:149100] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n11:170897] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n11:170897] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n11:170897] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n11:170897] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n11:170897] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n11:170897] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n11:170897] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n11:170897] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n11:170897] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n01:128829] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n01:128829] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n01:128829] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n07:180192] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n07:180192] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n07:180192] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n07:180192] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n07:180192] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n07:180192] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n09:128000] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n09:128000] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n09:128000] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n09:128000] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n09:128000] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n09:128000] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n11:133973] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n11:133973] *** End of error message *** +[a34n16:135788] *** Process received signal *** +[a34n16:135788] Signal: Segmentation fault (11) +[a34n16:135788] Signal code: (3) +[a34n16:135788] Failing at address: 0xb170000000000a0 +[a34n16:135788] [ 0] [0x1000000504d8] +[a34n16:135788] [ 1] [0x3fffec42eac0] +[a34n16:135788] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n16:135788] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n16:135788] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n16:135788] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n16:135788] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n16:135788] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n16:135788] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n16:135788] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n16:135788] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n16:135788] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n16:135788] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n16:135788] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n16:135788] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n16:135788] *** End of error message *** +[a02n03:38503] *** Process received signal *** +[a02n03:38503] Signal: Segmentation fault (11) +[a02n03:38503] Signal code: (3) +[a02n03:38503] Failing at address: 0x14170000000000a0 +[a02n03:38503] [ 0] [0x1000000504d8] +[a02n03:38503] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n01:48806] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n01:48806] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n01:48806] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n01:48806] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n01:48806] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n12:167678] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n11:160348] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n11:160348] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n11:160348] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n11:160348] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n13:114398] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n13:114398] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n12:12765] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n12:12765] [ 5] [a27n09:51334] *** Process received signal *** +[a27n09:51334] Signal: Segmentation fault (11) +[a27n09:51334] Signal code: (3) +[a27n09:51334] Failing at address: 0x14170000000000a0 +[a27n09:51334] [ 0] [0x1000000504d8] +[a27n09:51334] [ 1] [0x3ffff103be50] +[a27n09:51334] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n15:177058] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n15:177058] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n15:177058] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n15:177058] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n15:177058] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n15:177058] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n06:126374] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n18:179699] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n16:116526] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n16:116526] *** End of error message *** +[0x1000000504d8] +[a34n17:149907] [ 1] [0x3fffde645200] +[a34n17:149907] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n17:149907] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n17:149907] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n10:142930] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n16:157889] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n16:157889] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n16:157889] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n16:157889] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n16:157889] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n15:155433] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n15:155433] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n15:155433] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n05:135165] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n05:135165] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n05:135165] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n05:135165] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n05:135165] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n09:123419] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n09:123419] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n09:123419] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n09:123419] [ 8] [b03n02:06563] *** Process received signal *** +[b03n02:06563] Signal: Segmentation fault (11) +[b03n02:06563] Signal code: (3) +[b03n02:06563] Failing at address: 0x14170000000000a0 +[b03n02:06563] [ 0] [0x1000000504d8] +[b03n02:06563] [ 1] [0x3fffdc86efb0] +[b03n02:06563] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n02:06563] [ 3] [b22n08:118579] *** Process received signal *** +[b22n08:118579] Signal: Segmentation fault (11) +[b22n08:118579] Signal code: (3) +[b22n08:118579] Failing at address: 0x14170000000000a0 +[b22n08:118579] [ 0] [0x1000000504d8] +[b22n08:118579] [ 1] [0x3ffffc8b6d30] +[b22n08:118579] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n08:118579] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n08:118579] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n08:118579] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n08:118579] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n08:118579] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n08:118579] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n08:118579] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n08:118579] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n08:118579] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n08:118579] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n08:118579] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n08:118579] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n08:118579] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n07:118167] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n07:118167] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n07:118167] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n17:179115] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n17:179115] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n17:179115] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n17:179115] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n17:179115] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n17:179115] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n17:179115] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n17:179115] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n17:179115] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n17:179115] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n17:179115] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n14:128870] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n14:128870] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n13:122140] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n13:122140] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n13:122140] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n17:03843] [ 4] [b16n14:133636] *** Process received signal *** +[b16n14:133636] Signal: Segmentation fault (11) +[b16n14:133636] Signal code: (3) +[b16n14:133636] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n01:167297] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n01:167297] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n01:167297] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n01:167297] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n12:122964] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n12:122964] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n12:122964] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n12:122964] [ 9] [b02n14:169521] *** Process received signal *** +[b02n14:169521] Signal: Segmentation fault (11) +[b02n14:169521] Signal code: (3) +[b02n14:169521] Failing at address: 0xb170000000000a0 +[b02n14:169521] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n13:176367] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n13:176367] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n13:176367] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n13:176367] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n13:176367] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n09:03927] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n09:03927] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n09:03927] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n09:03927] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n09:03927] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n09:03927] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n09:03927] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n17:15874] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n17:15874] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n17:15874] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n01:141819] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n01:141819] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n01:141819] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n01:141819] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n17:138141] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n17:138141] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n17:138141] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n17:138141] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n06:01841] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n06:01841] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n10:176309] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n10:176309] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n03:177925] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n03:177925] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n03:177925] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n03:177925] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n07:138201] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n07:138201] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n07:138201] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n07:138201] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n08:08955] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n08:08955] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n08:08955] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n14:175714] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n16:01573] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n16:01573] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n16:01573] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n16:01573] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n14:159238] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n14:159238] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n06:43019] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n06:43019] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n02:166747] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n02:166747] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n02:166747] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n14:136661] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n14:136661] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n14:136661] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n14:136661] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n14:136661] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n14:136661] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n14:136661] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n14:136661] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n14:136661] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n18:126904] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n18:126904] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n04:169913] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n08:147822] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n08:147822] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n08:147822] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n08:147822] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n08:147822] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n07:132090] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n07:132090] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n10:67858] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n10:67858] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n10:67858] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n10:67858] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n10:67858] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n10:67858] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n02:164489] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n02:164489] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n02:164489] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n07:153178] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n07:153178] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n07:153178] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n07:153178] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n07:153178] *** End of error message *** +[a36n06:143010] *** Process received signal *** +[a36n06:143010] Signal: Segmentation fault (11) +[a36n06:143010] Signal code: (3) +[a36n06:143010] Failing at address: 0xb170000000000a0 +[a36n06:143010] [ 0] [0x1000000504d8] +[a36n06:143010] [ 1] [0x3ffff1d5d520] +[a36n06:143010] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n06:143010] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n07:131672] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n07:131672] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n07:131672] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n07:131672] [ 6] [0x1000000504d8] +[a29n02:163029] [ 1] [0x3fffd9a22590] +[a29n02:163029] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n02:163029] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n02:163029] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n02:163029] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n02:163029] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n02:163029] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n02:163029] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n02:163029] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n11:171965] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n11:171965] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n13:123708] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n13:123708] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n13:123708] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n01:128157] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n17:164296] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n17:164296] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n17:164296] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n17:164296] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n17:164296] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n17:164296] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n17:150927] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n17:150927] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n17:150927] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n17:150927] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n03:10291] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n09:121754] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n09:121754] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n09:121754] [14] [b11n11:152985] *** Process received signal *** +[b11n11:152985] Signal: Segmentation fault (11) +[b11n11:152985] Signal code: (3) +[b11n11:152985] Failing at address: 0xb170000000000a0 +[b11n11:152985] [ 0] [0x1000000504d8] +[b11n11:152985] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n17:156182] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n17:156182] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n17:156182] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n17:156182] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n17:156182] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n17:156182] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n17:156182] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n17:156182] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n17:156182] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n03:139011] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n03:139011] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n11:137874] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n11:137874] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n11:137874] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n11:137874] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n11:137874] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n11:137874] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n11:137874] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n11:152841] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n11:152841] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n11:152841] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n11:152841] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n11:152841] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n18:28216] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n18:28216] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n18:28216] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n18:28216] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n18:28216] [ 9] [b03n15:85942] *** Process received signal *** +[b03n15:85942] Signal: Segmentation fault (11) +[b03n15:85942] Signal code: (3) +[b03n15:85942] Failing at address: 0x5170000000000a0 +[b03n15:85942] [ 0] [0x1000000504d8] +[b03n15:85942] [ 1] [0x3fffcb770430] +[b03n15:85942] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n15:85942] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n15:85942] [ 4] [0x3fffc1f058a0] +[a28n11:01171] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n11:01171] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n11:01171] [ 4] [0x3fffde6aa3c0] +[b20n08:124722] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n08:124722] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n08:124722] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n09:173749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n13:01298] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n13:01298] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n13:01298] [12] [b06n08:174409] *** Process received signal *** +[b06n08:174409] Signal: Segmentation fault (11) +[b06n08:174409] Signal code: (3) +[b06n08:174409] Failing at address: 0x5170000000000a0 +[b06n08:174409] [ 0] [0x1000000504d8] +[b06n08:174409] [ 1] [b21n01:130409] *** Process received signal *** +[b21n01:130409] Signal: Segmentation fault (11) +[b21n01:130409] Signal code: (3) +[b21n01:130409] Failing at address: 0xb170000000000a0 +[b21n01:130409] [ 0] [0x1000000504d8] +[b21n01:130409] [ 1] [0x3fffed9c1410] +[b21n01:130409] [ 2] [0x1000000504d8] +[b21n10:130580] [ 1] [0x3fffec88d760] +[b21n10:130580] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n10:130580] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n10:130580] [ 4] [0x3fffcf5ed1a0] +[b12n02:143966] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n02:143966] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n02:143966] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n14:161617] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n14:161617] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n14:161617] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n14:161617] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n14:161617] [14] [0x1000000504d8] +[a30n16:153291] [ 1] [0x3fffd8eb65c0] +[a30n16:153291] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n16:153291] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n16:153291] [ 4] [b05n12:11146] *** Process received signal *** +[b05n12:11146] Signal: Segmentation fault (11) +[b05n12:11146] Signal code: (3) +[b05n12:11146] Failing at address: 0xb170000000000a0 +[b05n12:11146] [ 0] [0x1000000504d8] +[b05n12:11146] [ 1] [0x3fffda46a110] +[b05n12:11146] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n12:11146] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n12:11146] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n12:11146] [ 5] [0x3fffef2cfbb0] +[b21n06:122163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n06:122163] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n06:122163] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n06:122163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n06:122163] [ 6] [b14n09:119546] *** Process received signal *** +[b14n09:119546] Signal: Segmentation fault (11) +[b14n09:119546] Signal code: (3) +[b14n09:119546] Failing at address: 0xb170000000000a0 +[b14n09:119546] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n16:122494] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n16:122494] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n16:122494] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n16:122494] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n16:122494] [12] [b18n09:123814] *** Process received signal *** +[b18n09:123814] Signal: Segmentation fault (11) +[b18n09:123814] Signal code: (3) +[b18n09:123814] Failing at address: 0xb170000000000a0 +[b18n09:123814] [ 0] [0x1000000504d8] +[b18n09:123814] [ 1] [0x3fffc2b5e890] +[b18n09:123814] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n09:123814] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n09:123814] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n09:123814] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n09:123814] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n09:123814] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n09:123814] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n09:123814] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n09:123814] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n09:123814] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n09:123814] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n09:123814] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n09:123814] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n09:123814] *** End of error message *** +[b08n16:171916] *** Process received signal *** +[b08n16:171916] Signal: Segmentation fault (11) +[b08n16:171916] Signal code: (3) +[b08n16:171916] Failing at address: 0xb170000000000a0 +[b08n16:171916] [ 0] [a31n01:173782] *** Process received signal *** +[a31n01:173782] Signal: Segmentation fault (11) +[a31n01:173782] Signal code: (3) +[a31n01:173782] Failing at address: 0x14170000000000a0 +[a31n01:173782] [ 0] [0x1000000504d8] +[a31n01:173782] [ 1] [0x3ffff795f470] +[a31n01:173782] [ 2] [a30n12:148382] Signal: Segmentation fault (11) +[a30n12:148382] Signal code: (3) +[a30n12:148382] Failing at address: 0x14170000000000a0 +[a30n12:148382] [ 0] [0x1000000504d8] +[a30n12:148382] [ 1] [0x3fffd9ab6610] +[a30n12:148382] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n12:148382] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n12:148382] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n12:52454] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n12:52454] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n12:123234] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n12:123234] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n12:123234] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n17:119583] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n17:119583] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n01:145733] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n01:145733] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n01:145733] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n01:145733] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n12:156917] [10] [b21n02:120188] *** Process received signal *** +[b21n02:120188] Signal: Segmentation fault (11) +[b21n02:120188] Signal code: (3) +[b21n02:120188] Failing at address: 0xb170000000000a0 +[b21n02:120188] [ 0] [0x1000000504d8] +[b21n02:120188] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n04:176000] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n04:176000] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n04:176000] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n04:176000] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n15:148579] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n15:148579] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n15:148579] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n04:138159] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n04:138159] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n04:138159] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n04:138159] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n02:163321] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n02:163321] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n02:163321] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n02:163321] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n02:163321] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n17:164559] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n17:164559] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n17:164559] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n17:164559] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n17:164559] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n10:122208] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n10:122208] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n10:122208] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n10:122208] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n10:122208] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n10:122208] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n10:122208] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n10:122208] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n10:122208] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n10:122208] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n10:122208] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n10:122208] *** End of error message *** +[0x3ffffceeec60] +[b21n03:127629] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n03:127629] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n03:127629] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n18:163106] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n18:163106] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n18:163106] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n18:163106] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n18:163106] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n09:122069] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n09:122069] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n09:122069] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n09:122069] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n14:173895] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n14:173895] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n14:173895] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n14:173895] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n07:125473] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n07:125473] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n07:125473] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n07:125473] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n07:125473] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n07:125473] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n07:125473] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n07:125473] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n08:129888] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n08:129888] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n08:129888] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n08:129888] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n08:129888] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n08:163160] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n08:163160] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n08:163160] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n08:163160] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n04:164047] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n04:164047] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n04:164047] [ 8] [b09n03:168208] *** Process received signal *** +[b09n03:168208] Signal: Segmentation fault (11) +[b09n03:168208] Signal code: (3) +[b09n03:168208] Failing at address: 0x14170000000000a0 +[b09n03:168208] [ 0] [0x1000000504d8] +[b09n03:168208] [ 1] [0x3fffde93eb10] +[b09n03:168208] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n03:168208] [ 3] [b04n11:176297] *** Process received signal *** +[b04n11:176297] Signal: Segmentation fault (11) +[b04n11:176297] Signal code: (3) +[b04n11:176297] Failing at address: 0x14170000000000a0 +[b04n11:176297] [ 0] [0x1000000504d8] +[b04n11:176297] [ 1] [0x3fffe98bf910] +[b04n11:176297] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n11:176297] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n07:127354] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n07:127354] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n07:127354] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n07:127354] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n07:127354] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n09:152062] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n09:152062] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n09:152062] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n09:152062] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n09:152062] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n09:152062] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n09:152062] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n09:152062] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n09:152062] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n09:152062] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n09:152062] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n11:147712] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n11:147712] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n11:147712] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n11:147712] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n01:165165] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n01:165165] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n01:165165] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n01:165165] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n01:165165] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n18:145743] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n18:145743] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n18:145743] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n18:145743] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n18:145743] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n08:156371] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n08:156371] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n08:156371] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n08:156371] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n13:163194] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n13:163194] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n13:163194] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n13:163194] [ 9] [b07n15:166702] *** Process received signal *** +[b07n15:166702] Signal: Segmentation fault (11) +[b07n15:166702] Signal code: (3) +[b07n15:166702] Failing at address: 0x14170000000000a0 +[b07n15:166702] [ 0] [0x1000000504d8] +[b07n15:166702] [ 1] [0x3fffdef99ac0] +[b07n15:166702] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n15:166702] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n15:166702] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n15:166702] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n15:166702] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n15:166702] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n15:166702] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n15:166702] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n15:166702] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n15:166702] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n15:166702] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n15:166702] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n15:166702] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n15:166702] *** End of error message *** +[a27n06:60153] *** Process received signal *** +[a27n06:60153] Signal: Segmentation fault (11) +[a27n06:60153] Signal code: (3) +[a27n06:60153] Failing at address: 0x14170000000000a0 +[a27n06:60153] [ 0] [0x1000000504d8] +[a27n06:60153] [ 1] [0x3fffd011ca50] +[a27n06:60153] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n06:60153] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n06:60153] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n06:60153] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n06:60153] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n06:60153] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n06:60153] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n06:60153] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n06:60153] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n06:60153] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n06:60153] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n06:60153] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n06:60153] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n06:60153] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n02:155379] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n02:155379] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n13:165261] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n13:165261] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n13:165261] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n13:165261] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n15:137007] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n15:137007] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n15:137007] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n08:163099] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n08:163099] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n06:38118] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n06:38118] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n06:38118] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n06:38118] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n06:38118] *** End of error message *** +[b10n04:169409] *** Process received signal *** +[b10n04:169409] Signal: Segmentation fault (11) +[b10n04:169409] Signal code: (3) +[b10n04:169409] Failing at address: 0x14170000000000a0 +[b10n04:169409] [ 0] [0x1000000504d8] +[b10n04:169409] [ 1] [0x3fffd529fed0] +[b10n04:169409] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n04:169409] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n02:149226] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n02:149226] [ 7] [b04n05:169785] Signal: Segmentation fault (11) +[b04n05:169785] Signal code: (3) +[b04n05:169785] Failing at address: 0xb170000000000a0 +[b04n05:169785] [ 0] [0x1000000504d8] +[b04n05:169785] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n12:177119] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n12:177119] [ 5] [0x1000000504d8] +[a30n15:154383] [ 1] [0x3ffff0bf9980] +[a30n15:154383] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n15:154383] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n05:148377] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n05:148377] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n05:148377] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n05:148377] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n05:148377] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n05:148377] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n11:175595] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n11:175595] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n11:175595] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n11:175595] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n11:175595] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n11:175595] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n08:121687] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n08:121687] *** End of error message *** +[b02n03:39683] *** Process received signal *** +[b02n03:39683] Signal: Segmentation fault (11) +[b02n03:39683] Signal code: (3) +[b02n03:39683] Failing at address: 0xb170000000000a0 +[b02n03:39683] [ 0] [0x1000000504d8] +[b02n03:39683] [ 1] [0x3fffefc4ace0] +[b02n03:39683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n03:39683] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n12:115636] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n12:115636] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n12:115636] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n12:115636] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n06:177847] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n06:177847] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n06:177847] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n06:177847] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n06:177847] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n09:79006] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n09:79006] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n09:79006] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n09:79006] [10] [a35n03:135165] *** Process received signal *** +[a35n03:135165] Signal: Segmentation fault (11) +[a35n03:135165] Signal code: (3) +[a35n03:135165] Failing at address: 0x14170000000000a0 +[a35n03:135165] [ 0] [0x1000000504d8] +[a35n03:135165] [ 1] [0x3ffff499e110] +[a35n03:135165] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n03:135165] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n03:135165] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n03:135165] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n03:135165] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n03:135165] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n03:135165] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n11:120241] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n11:120241] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n11:120241] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n11:120241] [14] [0x3fffc01cfaa0] +[b04n06:10225] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n06:10225] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n06:10225] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n06:10225] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n06:10225] [ 6] [a32n11:152817] *** Process received signal *** +[a32n11:152817] Signal: Segmentation fault (11) +[a32n11:152817] Signal code: (3) +[a32n11:152817] Failing at address: 0xb170000000000a0 +[a32n11:152817] [ 0] [0x1000000504d8] +[a32n11:152817] [ 1] [0x3fffff5a6c10] +[a32n11:152817] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n11:152817] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n11:152817] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n12:154342] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n12:154342] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n12:154342] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n12:154342] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n12:154342] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n01:138004] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n01:138004] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n01:138004] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n01:138004] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n17:146794] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n17:146794] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n11:02684] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n11:02684] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n11:02684] [13] [b10n03:168932] *** Process received signal *** +[b10n03:168932] Signal: Segmentation fault (11) +[b10n03:168932] Signal code: (3) +[b10n03:168932] Failing at address: 0xb170000000000a0 +[b10n03:168932] [ 0] [0x3fffd87d4d80] +[b13n12:140121] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n12:140121] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n12:140121] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n12:140121] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n15:123041] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n15:123041] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n15:123041] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n15:123041] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n09:164363] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n09:164363] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n09:164363] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n09:164363] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n03:136668] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n03:136668] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n03:136668] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n14:163550] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n14:163550] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n14:163550] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n14:163550] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n14:163550] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n02:49105] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n02:49105] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n02:49105] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n02:49105] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n12:11418] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n12:11418] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n01:153682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n01:153682] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n17:119467] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n17:119467] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n17:119467] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n17:119467] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n17:119467] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n17:119467] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n15:161757] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n15:161757] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n15:161757] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n15:161757] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n14:21316] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n14:21316] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n14:21316] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n14:21316] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n14:21316] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n16:124793] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n03:167777] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n03:167777] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n03:167777] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n03:167777] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n03:167777] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n03:167777] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n03:167777] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n03:167777] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n03:167777] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n03:167777] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n03:167777] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n06:177004] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n06:177004] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n06:177004] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n06:177004] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n06:177004] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n01:65729] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n01:65729] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n01:65729] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n01:65729] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n01:65729] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n01:65729] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n01:65729] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n09:147790] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n09:147790] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n09:147790] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n09:147790] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n09:147790] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n09:147790] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n09:147790] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n09:147790] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n17:126527] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n17:126527] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n17:126527] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n17:126527] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n17:126527] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n03:125240] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n03:125240] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n06:154623] [ 5] [0x3fffebf63c20] +[b11n05:162566] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n05:162566] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n05:162566] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n04:169704] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n04:169704] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n04:169704] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n04:169704] [ 8] [b01n03:01354] *** Process received signal *** +[b01n03:01354] Signal: Segmentation fault (11) +[b01n03:01354] Signal code: (3) +[b01n03:01354] Failing at address: 0xb170000000000a0 +[b01n03:01354] [ 0] [0x1000000504d8] +[b01n03:01354] [ 1] [0x3fffe5a54270] +[b01n03:01354] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n03:01354] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n11:148675] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n11:148675] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n11:148675] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n11:148675] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n04:118344] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n04:118344] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n04:118344] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n04:118344] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n04:118344] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n04:54378] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n04:54378] *** End of error message *** +[a28n02:178475] Signal: Segmentation fault (11) +[a28n02:178475] Signal code: (3) +[a28n02:178475] Failing at address: 0xb170000000000a0 +[a28n02:178475] [ 0] [0x1000000504d8] +[a28n02:178475] [ 1] [0x3fffca425f10] +[a28n02:178475] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n02:178475] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n02:178475] [ 4] [0x3fffe67a9fb0] +[b17n02:127218] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n02:127218] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n02:127218] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n02:127218] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n02:122556] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n02:122556] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n02:122556] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n02:122556] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n02:123707] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n09:163513] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n10:161716] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n10:161716] [ 5] [b01n07:03460] *** Process received signal *** +[b01n07:03460] Signal: Segmentation fault (11) +[b01n07:03460] Signal code: (3) +[b01n07:03460] Failing at address: 0xb170000000000a0 +[b01n07:03460] [ 0] [0x1000000504d8] +[b01n07:03460] [ 1] [0x3ffff3590f60] +[b01n07:03460] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n07:03460] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n07:03460] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n07:03460] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n07:03460] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n07:03460] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n07:03460] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n07:03460] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n07:03460] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n07:03460] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n07:03460] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n07:03460] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n07:03460] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n07:03460] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n06:130543] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n05:131261] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n05:131261] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n05:131261] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n05:131261] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n08:172427] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n08:172427] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n08:172427] *** End of error message *** +[0x1000000504d8] +[b07n17:162358] [ 1] [0x3fffda4116a0] +[b07n17:162358] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n17:162358] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n17:162358] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n01:164963] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n06:133953] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n06:133953] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n06:133953] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n06:133953] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n06:133953] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n06:133953] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n06:133953] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n06:133953] [13] [b08n09:174691] *** Process received signal *** +[b08n09:174691] Signal: Segmentation fault (11) +[b08n09:174691] Signal code: (3) +[b08n09:174691] Failing at address: 0xb170000000000a0 +[b08n09:174691] [ 0] [0x1000000504d8] +[b08n09:174691] [ 1] [0x3fffea85b070] +[b08n09:174691] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n09:174691] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n09:174691] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n09:174691] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n09:174691] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n09:174691] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n09:174691] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n09:174691] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n09:174691] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n09:174691] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n09:174691] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n09:174691] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n09:174691] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n09:174691] *** End of error message *** +[a28n04:13079] *** Process received signal *** +[a28n04:13079] Signal: Segmentation fault (11) +[a28n04:13079] Signal code: (3) +[a28n04:13079] Failing at address: 0xb170000000000a0 +[a28n04:13079] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n05:135123] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n05:135123] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n05:135123] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n05:135123] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n05:135123] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n05:135123] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n05:135123] *** End of error message *** +[0x3fffc9543690] +[a32n03:151880] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n03:151880] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n03:151880] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n01:179263] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n01:179263] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n01:179263] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n01:179263] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n02:147885] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n02:147885] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n02:147885] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n02:147885] [ 9] [0x3fffc5f63f30] +[a28n08:175804] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n08:175804] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n08:175804] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n08:175804] [ 5] [b06n17:163819] *** Process received signal *** +[b06n17:163819] Signal: Segmentation fault (11) +[b06n17:163819] Signal code: (3) +[b06n17:163819] Failing at address: 0xb170000000000a0 +[b06n17:163819] [ 0] [0x1000000504d8] +[b06n17:163819] [ 1] [0x3fffcbf4c040] +[b06n17:163819] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n17:163819] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n05:54535] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n05:54535] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n05:54535] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n05:54535] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n17:118575] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n17:118575] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n17:118575] [ 8] [0x3fffcb9fe7b0] +[b08n01:171785] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n01:171785] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n01:171785] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n03:153849] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n03:153849] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n03:153849] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n03:153849] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n03:153849] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n03:153849] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n03:153849] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n03:153849] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n03:153849] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n03:153849] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n03:153849] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n03:153849] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n03:153849] *** End of error message *** +[a36n04:142254] *** Process received signal *** +[a36n04:142254] Signal: Segmentation fault (11) +[a36n04:142254] Signal code: (3) +[a36n04:142254] Failing at address: 0xb170000000000a0 +[a36n04:142254] [ 0] [0x1000000504d8] +[a36n04:142254] [ 1] [0x3fffc829df10] +[a36n04:142254] [ 2] [b08n11:164994] *** Process received signal *** +[b08n11:164994] Signal: Segmentation fault (11) +[b08n11:164994] Signal code: (3) +[b08n11:164994] Failing at address: 0x5170000000000a0 +[b08n11:164994] [ 0] [0x1000000504d8] +[b08n11:164994] [ 1] [0x3fffc5138b40] +[b08n11:164994] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n11:164994] [ 3] [0x3fffdf086990] +[b21n05:130803] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n05:130803] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n05:130803] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n05:130803] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n05:130803] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n05:130803] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n05:130803] [ 8] [b21n18:118681] *** Process received signal *** +[b21n18:118681] Signal: Segmentation fault (11) +[b21n18:118681] Signal code: (3) +[b21n18:118681] Failing at address: 0xb170000000000a0 +[b21n18:118681] [ 0] [0x1000000504d8] +[b21n18:118681] [ 1] [0x3fffd35d1fd0] +[b21n18:118681] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n18:118681] [ 3] [a32n05:154189] *** Process received signal *** +[a32n05:154189] Signal: Segmentation fault (11) +[a32n05:154189] Signal code: (3) +[a32n05:154189] Failing at address: 0x5170000000000a0 +[a32n05:154189] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n02:121683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n02:121683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n02:121683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n02:121683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n02:121683] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n01:141341] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n01:141341] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n01:141341] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n01:141341] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n10:164940] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n10:164940] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n10:164940] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n10:164940] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n06:162920] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n06:162920] *** End of error message *** +[b15n16:121929] [ 0] [0x1000000504d8] +[b15n16:121929] [ 1] [0x3fffc9f22430] +[b15n16:121929] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n16:121929] [ 3] [0x3fffe90e7c00] +[a36n14:152036] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n14:152036] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n14:152036] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n14:152036] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n14:152036] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n14:152036] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n14:152036] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n14:152036] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n01:158305] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n04:123054] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n04:123054] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n16:171168] *** End of error message *** +[ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n08:164752] [ 6] [b22n06:133056] *** Process received signal *** +[b22n06:133056] Signal: Segmentation fault (11) +[b22n06:133056] Signal code: (3) +[b22n06:133056] Failing at address: 0x14170000000000a0 +[b22n06:133056] [ 0] [0x1000000504d8] +[b22n06:133056] [ 1] [0x3fffe216e9a0] +[b22n06:133056] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n06:133056] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n06:133056] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n06:133056] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n06:133056] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n06:133056] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n06:133056] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n06:133056] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n06:133056] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n06:133056] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n06:133056] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n06:133056] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n06:133056] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n06:133056] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n15:169494] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n15:169494] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n15:169494] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n15:169494] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n15:169494] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n15:169494] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n15:169494] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n15:169494] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n15:169494] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n15:169494] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n15:169494] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n15:169494] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n15:169494] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n04:04849] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n04:04849] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n04:04849] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n04:04849] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n01:132063] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n01:132063] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n01:132063] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n01:132063] [12] [b15n04:133904] *** Process received signal *** +[b15n04:133904] Signal: Segmentation fault (11) +[b15n04:133904] Signal code: (3) +[b15n04:133904] Failing at address: 0xb170000000000a0 +[b15n04:133904] [ 0] [0x1000000504d8] +[b15n04:133904] [ 1] [0x3fffe7e54e90] +[b15n04:133904] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n04:133904] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n04:133904] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n04:133904] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n04:133904] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n04:133904] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n04:133904] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n04:133904] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n04:133904] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n04:133904] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n04:133904] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n04:133904] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n04:133904] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n04:133904] *** End of error message *** +[ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n06:132106] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n06:132106] [10] [b23n02:116884] *** Process received signal *** +[b23n02:116884] Signal: Segmentation fault (11) +[b23n02:116884] Signal code: (3) +[b23n02:116884] Failing at address: 0xb170000000000a0 +[b23n02:116884] [ 0] [0x1000000504d8] +[b23n02:116884] [ 1] [0x3ffff3a68590] +[b23n02:116884] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n02:116884] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n06:120734] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n06:120734] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n06:120734] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n06:120734] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n06:120734] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n06:120734] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n06:120734] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n05:02908] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n05:02908] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n05:02908] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n05:02908] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n05:02908] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n05:02908] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n05:02908] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n05:02908] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n05:02908] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n06:171504] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n06:171504] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n06:171504] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n06:171504] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n03:130345] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n03:130345] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n03:130345] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n03:130345] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n03:130345] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n07:03372] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n07:03372] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n07:03372] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n07:03372] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n07:03372] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n07:03372] [ 9] [b10n18:162420] *** Process received signal *** +[b10n18:162420] Signal: Segmentation fault (11) +[b10n18:162420] Signal code: (3) +[b10n18:162420] Failing at address: 0xb170000000000a0 +[b10n18:162420] [ 0] [0x1000000504d8] +[b10n18:162420] [ 1] [0x3fffd37fff00] +[b10n18:162420] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n18:162420] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n18:162420] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n18:162420] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n18:162420] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n18:162420] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n18:162420] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n18:162420] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n18:162420] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n18:162420] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n18:162420] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n18:162420] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n18:162420] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n18:162420] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n10:02712] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n10:02712] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n08:121434] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n08:121434] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n05:122271] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n05:122271] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n05:122271] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n05:122271] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n05:122271] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n05:173080] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n02:139023] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n02:139023] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n02:139023] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n02:139023] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n02:139023] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n06:163677] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n06:163677] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n06:163677] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n11:160798] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n11:160798] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n11:160798] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n11:160798] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n01:128004] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n01:128004] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n01:128004] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n01:128004] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n01:128004] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n12:138737] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n05:120333] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n05:120333] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n05:120333] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n05:120333] [12] [b14n04:126857] *** Process received signal *** +[b14n04:126857] Signal: Segmentation fault (11) +[b14n04:126857] Signal code: (3) +[b14n04:126857] Failing at address: 0x14170000000000a0 +[b14n04:126857] [ 0] [b10n16:164550] *** Process received signal *** +[b10n16:164550] Signal: Segmentation fault (11) +[b10n16:164550] Signal code: (3) +[b10n16:164550] Failing at address: 0x14170000000000a0 +[b10n16:164550] [ 0] [0x1000000504d8] +[b10n16:164550] [ 1] [0x3ffff8599980] +[b10n16:164550] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n16:164550] [ 3] [a32n15:139054] *** Process received signal *** +[a32n15:139054] Signal: Segmentation fault (11) +[a32n15:139054] Signal code: (3) +[a32n15:139054] Failing at address: 0xb170000000000a0 +[a32n15:139054] [ 0] [0x1000000504d8] +[a32n15:139054] [ 1] [0x3ffffd485cc0] +[a32n15:139054] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n15:139054] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n15:139054] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n15:139054] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n15:139054] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n15:139054] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n15:139054] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n15:139054] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n15:139054] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n15:139054] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n15:139054] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n15:139054] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n15:139054] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n15:139054] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n05:126781] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n05:126781] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n05:126781] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n05:126781] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n05:126781] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n07:124352] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n07:124352] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n07:124352] [13] [b19n17:123234] *** Process received signal *** +[b19n17:123234] Signal: Segmentation fault (11) +[b19n17:123234] Signal code: (3) +[b19n17:123234] Failing at address: 0x14170000000000a0 +[b19n17:123234] [ 0] [0x1000000504d8] +[b19n17:123234] [ 1] [0x3fffd20972b0] +[b19n17:123234] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n17:123234] [ 3] [b05n13:10196] *** Process received signal *** +[b05n13:10196] Signal: Segmentation fault (11) +[b05n13:10196] Signal code: (3) +[b05n13:10196] Failing at address: 0x14170000000000a0 +[b05n13:10196] [ 0] [0x1000000504d8] +[b05n13:10196] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n09:155756] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n01:165127] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n01:165127] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n01:165127] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n01:165127] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n01:165127] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n17:176414] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n17:176414] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n06:157789] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n06:157789] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n06:157789] [ 8] [b23n11:117472] *** Process received signal *** +[b23n11:117472] Signal: Segmentation fault (11) +[b23n11:117472] Signal code: (3) +[b23n11:117472] Failing at address: 0xb170000000000a0 +[b23n11:117472] [ 0] [0x3fffc18dcc90] +[b11n13:153675] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n13:153675] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n13:153675] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n13:153675] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n13:153675] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n11:120580] *** End of error message *** +[a35n06:138763] *** Process received signal *** +[a35n06:138763] Signal: Segmentation fault (11) +[a35n06:138763] Signal code: (3) +[a35n06:138763] Failing at address: 0xb170000000000a0 +[a35n06:138763] [ 0] [0x1000000504d8] +[a35n06:138763] [ 1] [0x3ffffcd6ede0] +[a35n06:138763] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n06:138763] [ 3] [a33n17:168154] *** Process received signal *** +[a33n17:168154] Signal: Segmentation fault (11) +[a33n17:168154] Signal code: (3) +[a33n17:168154] Failing at address: 0xb170000000000a0 +[a33n17:168154] [ 0] [0x1000000504d8] +[a33n17:168154] [ 1] [b08n06:161723] *** Process received signal *** +[b08n06:161723] Signal: Segmentation fault (11) +[b08n06:161723] Signal code: (3) +[b08n06:161723] Failing at address: 0x14170000000000a0 +[b08n06:161723] [ 0] [0x1000000504d8] +[b08n06:161723] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n06:117526] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n06:117526] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n07:152083] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n07:152083] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n07:152083] [13] [b07n10:16973] *** Process received signal *** +[b07n10:16973] Signal: Segmentation fault (11) +[b07n10:16973] Signal code: (3) +[b07n10:16973] Failing at address: 0x14170000000000a0 +[b07n10:16973] [ 0] [0x1000000504d8] +[b07n10:16973] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n11:03985] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n11:03985] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n11:03985] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n11:03985] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n11:03985] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n11:03985] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n03:120667] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n03:120667] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n03:120667] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n03:120667] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n03:120667] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n03:120667] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n03:120667] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n03:120667] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n03:120667] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n03:120667] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n07:160802] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n07:160802] *** End of error message *** +[a27n03:46690] [ 0] [0x1000000504d8] +[a27n03:46690] [ 1] [0x3ffff34f1c40] +[a27n03:46690] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n03:46690] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n03:46690] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n03:46690] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n03:46690] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n03:46690] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n04:158636] *** End of error message *** +[0x1000000504d8] +[b01n12:178599] [ 1] [0x3fffdd38c410] +[b01n12:178599] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n12:178599] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n12:178599] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n03:154295] *** End of error message *** +[b08n14:178325] *** Process received signal *** +[b08n14:178325] Signal: Segmentation fault (11) +[b08n14:178325] Signal code: (3) +[b08n14:178325] Failing at address: 0x14170000000000a0 +[b08n14:178325] [ 0] [0x1000000504d8] +[b08n14:178325] [ 1] [0x3ffff8664780] +[b08n14:178325] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n14:178325] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n09:119634] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n09:119634] *** End of error message *** +[a35n04:145748] *** Process received signal *** +[a35n04:145748] Signal: Segmentation fault (11) +[a35n04:145748] Signal code: (3) +[a35n04:145748] Failing at address: 0x14170000000000a0 +[a35n04:145748] [ 0] [0x1000000504d8] +[a35n04:145748] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n13:171110] *** End of error message *** +[b22n05:123937] *** Process received signal *** +[b22n05:123937] Signal: Segmentation fault (11) +[b22n05:123937] Signal code: (3) +[b22n05:123937] Failing at address: 0x14170000000000a0 +[b22n05:123937] [ 0] [0x1000000504d8] +[b22n05:123937] [ 1] [0x3ffff88ada20] +[b22n05:123937] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n05:123937] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n05:123937] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n05:123937] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n05:123937] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n05:123937] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n05:123937] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n05:123937] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n05:123937] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n05:123937] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n05:123937] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n05:123937] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n05:123937] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n05:123937] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n05:139433] *** End of error message *** +[b08n10:11491] *** Process received signal *** +[b08n10:11491] Signal: Segmentation fault (11) +[b08n10:11491] Signal code: (3) +[b08n10:11491] Failing at address: 0x14170000000000a0 +[b08n10:11491] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n07:168964] *** End of error message *** +[a28n18:179170] *** Process received signal *** +[a28n18:179170] Signal: Segmentation fault (11) +[a28n18:179170] Signal code: (3) +[a28n18:179170] Failing at address: 0x14170000000000a0 +[a28n18:179170] [ 0] [0x1000000504d8] +[a28n18:179170] [ 1] [0x3fffd6130090] +[a28n18:179170] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n18:179170] [ 3] [b13n11:134517] *** Process received signal *** +[b13n11:134517] Signal: Segmentation fault (11) +[b13n11:134517] Signal code: (3) +[b13n11:134517] Failing at address: 0x14170000000000a0 +[b13n11:134517] [ 0] [0x1000000504d8] +[b13n11:134517] [ 1] [0x3fffee7ca810] +[b13n11:134517] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n11:134517] [ 3] [a29n17:169735] *** Process received signal *** +[a29n17:169735] Signal: Segmentation fault (11) +[a29n17:169735] Signal code: (3) +[a29n17:169735] Failing at address: 0xb170000000000a0 +[a29n17:169735] [ 0] [0x1000000504d8] +[a29n17:169735] [ 1] [0x3fffcd008db0] +[a29n17:169735] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n17:169735] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n17:169735] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n13:123516] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n13:123516] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n04:44755] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n04:44755] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n12:123559] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n03:06824] *** End of error message *** +[a36n05:139913] *** Process received signal *** +[a36n05:139913] Signal: Segmentation fault (11) +[a36n05:139913] Signal code: (3) +[a36n05:139913] Failing at address: 0x5170000000000a0 +[a36n05:139913] [ 0] [a30n05:156008] *** Process received signal *** +[a30n05:156008] Signal: Segmentation fault (11) +[a30n05:156008] Signal code: (3) +[a30n05:156008] Failing at address: 0x14170000000000a0 +[a30n05:156008] [ 0] [0x1000000504d8] +[a30n05:156008] [ 1] [0x3ffffe4a3060] +[a30n05:156008] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n05:156008] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n10:169208] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n05:174781] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n13:123054] *** End of error message *** +[a28n14:08399] *** Process received signal *** +[a28n14:08399] Signal: Segmentation fault (11) +[a28n14:08399] Signal code: (3) +[a28n14:08399] Failing at address: 0xb170000000000a0 +[a28n14:08399] [ 0] [0x1000000504d8] +[a28n14:08399] [ 1] [0x3ffff7ba5630] +[a28n14:08399] [ 2] [b15n14:133969] *** Process received signal *** +[b15n14:133969] Signal: Segmentation fault (11) +[b15n14:133969] Signal code: (3) +[b15n14:133969] Failing at address: 0xb170000000000a0 +[b15n14:133969] [ 0] [0x1000000504d8] +[b15n14:133969] [ 1] [0x3ffffd6784d0] +[b15n14:133969] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n14:133969] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n14:133969] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n14:133969] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n14:133969] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n14:133969] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n14:133969] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n14:133969] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n14:133969] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n14:133969] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n14:133969] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n14:133969] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n14:133969] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n14:133969] *** End of error message *** +[0x1000000504d8] +[a30n13:154230] [ 1] [0x3fffe2b0c8a0] +[a30n13:154230] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n13:154230] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n13:154230] [ 4] [a30n01:163390] [ 0] [0x1000000504d8] +[a30n01:163390] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n08:141873] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n08:141873] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n08:141873] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n08:141873] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n08:141873] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n04:03147] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n04:03147] [ 5] [b22n10:118390] *** Process received signal *** +[b22n10:118390] Signal: Segmentation fault (11) +[b22n10:118390] Signal code: (3) +[b22n10:118390] Failing at address: 0x14170000000000a0 +[b22n10:118390] [ 0] [b07n12:160962] *** Process received signal *** +[b07n12:160962] Signal: Segmentation fault (11) +[b07n12:160962] Signal code: (3) +[b07n12:160962] Failing at address: 0x14170000000000a0 +[b07n12:160962] [ 0] [0x1000000504d8] +[b07n12:160962] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n16:53796] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n08:176679] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n08:176679] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n08:176679] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n08:176679] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n08:176679] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n08:176679] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n08:176679] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n01:96754] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n01:96754] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n15:120133] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n15:120133] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n15:166120] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n15:166120] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n15:166120] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n15:166120] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n15:166120] *** End of error message *** +[b04n09:12245] *** Process received signal *** +[b04n09:12245] Signal: Segmentation fault (11) +[b04n09:12245] Signal code: (3) +[b04n09:12245] Failing at address: 0x14170000000000a0 +[b04n09:12245] [ 0] [0x1000000504d8] +[b04n09:12245] [ 1] [0x3fffca23a0d0] +[b04n09:12245] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n09:12245] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n11:130804] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n05:120231] *** End of error message *** +[a28n06:179454] *** Process received signal *** +[a28n06:179454] Signal: Segmentation fault (11) +[a28n06:179454] Signal code: (3) +[a28n06:179454] Failing at address: 0xb170000000000a0 +[a28n06:179454] [ 0] [0x1000000504d8] +[a28n06:179454] [ 1] [0x3ffff52351a0] +[a28n06:179454] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n06:179454] [ 3] [0x1000000504d8] +[b03n01:170768] [ 1] [0x3ffff38f14a0] +[b03n01:170768] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n01:170768] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n01:170768] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n01:170768] [ 5] [b13n18:128544] *** Process received signal *** +[b13n18:128544] Signal: Segmentation fault (11) +[b13n18:128544] Signal code: (3) +[b13n18:128544] Failing at address: 0xb170000000000a0 +[b13n18:128544] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n08:159026] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n08:159026] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n08:159026] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n08:159026] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n08:159026] *** End of error message *** +[b07n03:172650] *** Process received signal *** +[b07n03:172650] Signal: Segmentation fault (11) +[b07n03:172650] Signal code: (3) +[b07n03:172650] Failing at address: 0x14170000000000a0 +[b07n03:172650] [ 0] [0x1000000504d8] +[b07n03:172650] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n10:140415] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n10:140415] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n10:140415] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n10:140415] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n10:140415] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n10:140415] [10] [0x1000000504d8] +[b11n16:154183] [ 1] [0x3fffe68226f0] +[b11n16:154183] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n16:154183] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n16:154183] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n12:145496] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n12:145496] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n12:145496] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n12:145496] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n12:145496] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n12:145496] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n07:165090] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n07:165090] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n07:165090] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n07:165090] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n07:165090] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n10:126611] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n10:126611] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n10:126611] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n10:126611] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n10:126611] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n10:126611] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n10:126611] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n10:126611] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n10:126611] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n10:126611] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n01:124277] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n01:124277] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n01:124277] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n01:124277] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n01:124277] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n01:124277] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n01:124277] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n06:119114] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n06:119114] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n06:119114] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n06:119114] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n11:156154] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n11:156154] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n11:156154] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n11:156154] [13] [b20n02:122181] Signal: Segmentation fault (11) +[b20n02:122181] Signal code: (3) +[b20n02:122181] Failing at address: 0x14170000000000a0 +[b20n02:122181] [ 0] [0x1000000504d8] +[b20n02:122181] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n08:176175] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n08:176175] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n08:176175] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n08:176175] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n08:176175] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n08:176175] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n08:176175] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n05:04520] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n05:04520] *** End of error message *** +[a32n17:138613] *** Process received signal *** +[a32n17:138613] Signal: Segmentation fault (11) +[a32n17:138613] Signal code: (3) +[a32n17:138613] Failing at address: 0x5170000000000a0 +[a32n17:138613] [ 0] [0x1000000504d8] +[a32n17:138613] [ 1] [0x3fffe19f80c0] +[a32n17:138613] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n17:138613] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n17:138613] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n17:138613] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n17:138613] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n17:138613] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n17:138613] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n17:138613] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n17:138613] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n17:138613] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n17:138613] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n17:138613] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n17:138613] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n17:138613] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n15:139747] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n15:139747] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n15:139747] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n15:139747] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n15:139747] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n10:155396] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n10:169685] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n10:169685] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n13:133623] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n13:133623] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n13:133623] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n13:133623] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n13:133623] *** End of error message *** +[0x1000000504d8] +[a30n18:10723] [ 1] [0x3fffe8fda9f0] +[a30n18:10723] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n18:10723] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n18:10723] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n08:127446] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n08:127446] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n08:127446] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n08:127446] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n14:126465] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n14:126465] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n14:126465] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n14:126465] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n09:168244] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n09:168244] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n09:168244] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n09:168244] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n09:168244] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n09:168244] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n09:168244] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n09:168244] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n09:168244] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n09:168244] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n14:152747] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n10:10923] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n10:10923] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n10:10923] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n10:10923] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n05:173317] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n05:173317] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n05:173317] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n05:173317] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n05:173317] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n05:173317] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n09:160373] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n09:160373] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n09:160373] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n09:160373] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n09:160373] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n09:160373] [ 9] [b15n15:122084] *** Process received signal *** +[b15n15:122084] Signal: Segmentation fault (11) +[b15n15:122084] Signal code: (3) +[b15n15:122084] Failing at address: 0xb170000000000a0 +[b15n15:122084] [ 0] [0x1000000504d8] +[b15n15:122084] [ 1] [0x3fffc277ceb0] +[b15n15:122084] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n15:122084] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n15:122084] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n03:09641] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n03:09641] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n03:09641] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n03:09641] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n01:121742] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n01:121742] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n01:121742] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n01:121742] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n01:121742] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n01:121742] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n02:74427] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n02:74427] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n02:74427] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n02:74427] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n02:74427] *** End of error message *** +[a02n10:135850] *** Process received signal *** +[a02n10:135850] Signal: Segmentation fault (11) +[a02n10:135850] Signal code: (3) +[a02n10:135850] Failing at address: 0xb170000000000a0 +[a02n10:135850] [ 0] [0x1000000504d8] +[a02n10:135850] [ 1] [b19n10:120978] *** Process received signal *** +[b19n10:120978] Signal: Segmentation fault (11) +[b19n10:120978] Signal code: (3) +[b19n10:120978] Failing at address: 0x14170000000000a0 +[b19n10:120978] [ 0] [0x1000000504d8] +[b19n10:120978] [ 1] [0x3fffe68bab70] +[b19n10:120978] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n10:120978] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n07:172877] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n17:122356] *** End of error message *** +[0x1000000504d8] +[b06n03:164494] [ 1] [0x3fffe14035d0] +[b06n03:164494] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n03:164494] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n03:164494] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n03:164494] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n02:125266] *** End of error message *** +[b19n04:130748] *** Process received signal *** +[b19n04:130748] Signal: Segmentation fault (11) +[b19n04:130748] Signal code: (3) +[b19n04:130748] Failing at address: 0x14170000000000a0 +[b19n04:130748] [ 0] [0x1000000504d8] +[b19n04:130748] [ 1] [0x3fffdfbac1e0] +[b19n04:130748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n04:130748] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n04:130748] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n04:130748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n04:130748] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n04:130748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n04:130748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n04:130748] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n04:130748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n04:130748] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n04:130748] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n04:130748] [13] [0x1000000504d8] +[b19n12:122875] [ 1] [0x3ffff5aeff10] +[b19n12:122875] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n12:122875] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n12:122875] [ 4] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n16:124961] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n16:124961] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n16:124961] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n13:155850] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n13:155850] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n13:155850] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n13:155850] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n13:155850] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n05:149465] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n05:149465] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n05:149465] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n05:149465] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n02:174058] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n02:174058] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n02:174058] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n02:174058] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n02:174058] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n02:174058] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n02:174058] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n02:174058] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n02:174058] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n02:174058] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n02:174058] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n07:45306] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n07:45306] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n07:45306] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n07:45306] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n07:45306] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n07:45306] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n07:45306] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n07:45306] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n07:45306] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n07:45306] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n07:45306] *** End of error message *** +[a29n12:160191] *** Process received signal *** +[a29n12:160191] Signal: Segmentation fault (11) +[a29n12:160191] Signal code: (3) +[a29n12:160191] Failing at address: 0xb170000000000a0 +[a29n12:160191] [ 0] [0x1000000504d8] +[a29n12:160191] [ 1] [0x3ffff405c890] +[a29n12:160191] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n12:160191] [ 3] [0x1000000504d8] +[b05n09:106787] [ 1] [0x3fffe3444c60] +[b05n09:106787] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n09:106787] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n09:106787] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n09:106787] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n09:106787] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n09:106787] [ 7] [0x3fffea1fa060] +[b05n14:177512] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n14:177512] [ 3] [b18n02:125373] *** Process received signal *** +[b18n02:125373] Signal: Segmentation fault (11) +[b18n02:125373] Signal code: (3) +[b18n02:125373] Failing at address: 0x14170000000000a0 +[b18n02:125373] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n05:176910] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n05:176910] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n05:176910] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n05:176910] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n09:170658] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n09:170658] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n10:122757] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n10:122757] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n10:122757] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n10:122757] [ 8] [0x1000000504d8] +[a02n08:146116] [ 1] [0x3fffeda43750] +[a02n08:146116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n08:146116] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n08:146116] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n04:156413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n04:156413] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n04:156413] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n04:156413] [10] [a31n07:156489] *** Process received signal *** +[a31n07:156489] Signal: Segmentation fault (11) +[a31n07:156489] Signal code: (3) +[a31n07:156489] Failing at address: 0x14170000000000a0 +[a31n07:156489] [ 0] [0x1000000504d8] +[a31n07:156489] [ 1] [0x3fffec174220] +[a31n07:156489] [ 2] [b17n03:135038] *** Process received signal *** +[b17n03:135038] Signal: Segmentation fault (11) +[b17n03:135038] Signal code: (3) +[b17n03:135038] Failing at address: 0x5170000000000a0 +[b17n03:135038] [ 0] [0x1000000504d8] +[b17n03:135038] [ 1] [0x3fffed1cb740] +[b17n03:135038] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n03:135038] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n08:165282] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n08:165282] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n08:165282] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n08:165282] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n17:163177] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n17:163177] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n07:02841] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n07:02841] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n14:126118] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n14:126118] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n14:126118] [ 8] [b06n10:176264] *** Process received signal *** +[b06n10:176264] Signal: Segmentation fault (11) +[b06n10:176264] Signal code: (3) +[b06n10:176264] Failing at address: 0xb170000000000a0 +[b06n10:176264] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n04:136997] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n04:136997] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n04:136997] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n04:136997] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n04:136997] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n04:136997] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n04:136997] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n04:136997] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n04:136997] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n04:136997] [14] [b15n08:118550] *** Process received signal *** +[b15n08:118550] Signal: Segmentation fault (11) +[b15n08:118550] Signal code: (3) +[b15n08:118550] Failing at address: 0x14170000000000a0 +[b15n08:118550] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n03:126186] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n03:126186] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n03:126186] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n03:126186] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n03:147537] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n15:165473] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n15:165473] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n15:165473] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n15:165473] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n15:165473] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n03:68831] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n03:68831] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n03:68831] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n03:68831] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n03:68831] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n03:68831] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n03:68831] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n11:134748] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n11:134748] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n11:134748] [14] [a35n12:145265] *** Process received signal *** +[a35n12:145265] Signal: Segmentation fault (11) +[a35n12:145265] Signal code: (3) +[a35n12:145265] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n09:128492] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n09:128492] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n09:128492] [ 6] [a34n13:137747] *** Process received signal *** +[a34n13:137747] Signal: Segmentation fault (11) +[a34n13:137747] Signal code: (3) +[a34n13:137747] Failing at address: 0x14170000000000a0 +[a34n13:137747] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n14:122444] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n14:122444] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n14:122444] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n14:122444] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n14:122444] *** End of error message *** +[b15n01:123569] *** Process received signal *** +[b15n01:123569] Signal: Segmentation fault (11) +[b15n01:123569] Signal code: (3) +[b15n01:123569] Failing at address: 0x14170000000000a0 +[b15n01:123569] [ 0] [0x1000000504d8] +[b15n01:123569] [ 1] [0x3fffdf9efd60] +[b15n01:123569] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n01:123569] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n05:173823] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n05:173823] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n05:173823] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n05:173823] [14] [b15n03:119830] *** Process received signal *** +[b15n03:119830] Signal: Segmentation fault (11) +[b15n03:119830] Signal code: (3) +[b15n03:119830] Failing at address: 0x14170000000000a0 +[b15n03:119830] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n07:125450] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n07:125450] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n07:125450] [10] [b17n08:124115] *** Process received signal *** +[b17n08:124115] Signal: Segmentation fault (11) +[b17n08:124115] Signal code: (3) +[b17n08:124115] Failing at address: 0xb170000000000a0 +[b17n08:124115] [ 0] [0x1000000504d8] +[b17n08:124115] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n13:165386] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n13:165386] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n13:165386] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n11:128954] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n11:128954] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n11:128954] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n11:128954] [10] [a34n06:148311] *** Process received signal *** +[a34n06:148311] Signal: Segmentation fault (11) +[a34n06:148311] Signal code: (3) +[a34n06:148311] Failing at address: 0x14170000000000a0 +[a34n06:148311] [ 0] [0x1000000504d8] +[a34n06:148311] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n04:177597] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n04:177597] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n04:177597] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n04:177597] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n03:124682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n03:124682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n03:124682] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n03:124682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n03:124682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n03:124682] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n07:163254] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n07:163254] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n07:163254] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n07:163254] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n07:163254] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n07:163254] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n07:163254] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n07:163254] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n07:163254] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n07:163254] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n07:163254] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n07:163254] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n12:120405] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n12:120405] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n12:120405] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n12:120405] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n16:129236] [ 6] [b12n14:139842] *** Process received signal *** +[b12n14:139842] Signal: Segmentation fault (11) +[b12n14:139842] Signal code: (3) +[b12n14:139842] Failing at address: 0x14170000000000a0 +[b12n14:139842] [ 0] [0x1000000504d8] +[b12n14:139842] [ 1] [0x3fffd43bd2f0] +[b12n14:139842] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n14:139842] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n12:168641] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n12:168641] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n12:168641] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n12:168641] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n12:168641] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n12:168641] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n12:168641] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n12:168641] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n12:168641] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n12:168641] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n12:168641] *** End of error message *** +[b03n14:177221] *** Process received signal *** +[b03n14:177221] Signal: Segmentation fault (11) +[b03n14:177221] Signal code: (3) +[b03n14:177221] Failing at address: 0x5170000000000a0 +[b03n14:177221] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n14:116411] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n14:116411] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n14:116411] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n14:116411] [12] [b18n18:121306] *** Process received signal *** +[b18n18:121306] Signal: Segmentation fault (11) +[b18n18:121306] Signal code: (3) +[b18n18:121306] Failing at address: 0xb170000000000a0 +[b18n18:121306] [ 0] [0x1000000504d8] +[b18n18:121306] [ 1] [0x3ffff0b275d0] +[b18n18:121306] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n18:121306] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n04:136608] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n04:136608] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n04:136608] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n04:136608] [ 9] [b03n05:05605] *** Process received signal *** +[b03n05:05605] Signal: Segmentation fault (11) +[b03n05:05605] Signal code: (3) +[b03n05:05605] Failing at address: 0x14170000000000a0 +[b03n05:05605] [ 0] [0x1000000504d8] +[b03n05:05605] [ 1] [0x3fffcbafdb00] +[b03n05:05605] [ 2] [b23n04:117352] *** Process received signal *** +[b23n04:117352] Signal: Segmentation fault (11) +[b23n04:117352] Signal code: (3) +[b23n04:117352] Failing at address: 0xb170000000000a0 +[b23n04:117352] [ 0] [b01n02:01428] *** Process received signal *** +[b01n02:01428] Signal: Segmentation fault (11) +[b01n02:01428] Signal code: (3) +[b01n02:01428] Failing at address: 0x14170000000000a0 +[b01n02:01428] [ 0] [0x1000000504d8] +[b01n02:01428] [ 1] [0x3ffff69e15f0] +[b01n02:01428] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n02:01428] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n02:01428] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n02:01428] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n02:01428] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n02:01428] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n02:01428] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n02:01428] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n02:01428] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n02:01428] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n02:01428] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n02:01428] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n02:01428] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n02:01428] *** End of error message *** +[a28n12:177493] *** Process received signal *** +[a28n12:177493] Signal: Segmentation fault (11) +[a28n12:177493] Signal code: (3) +[a28n12:177493] Failing at address: 0xb170000000000a0 +[a28n12:177493] [ 0] [0x1000000504d8] +[a28n12:177493] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n13:177638] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n13:177638] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n13:177638] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n13:177638] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n05:53372] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n05:53372] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n05:53372] [ 8] [b13n09:143897] *** Process received signal *** +[b13n09:143897] Signal: Segmentation fault (11) +[b13n09:143897] Signal code: (3) +[b13n09:143897] Failing at address: 0xb170000000000a0 +[b13n09:143897] [ 0] [0x1000000504d8] +[b13n09:143897] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n08:144854] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n08:144854] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n08:144854] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n07:156139] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n07:156139] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n07:156139] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n07:156139] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n09:166573] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n09:166573] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n09:166573] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n09:166573] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n08:138175] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n08:138175] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n08:138175] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n08:138175] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n13:54241] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n13:54241] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n13:54241] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n13:54241] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n13:54241] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n13:54241] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n12:119805] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n12:119805] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n12:119805] *** End of error message *** +[b20n05:121821] *** Process received signal *** +[b20n05:121821] Signal: Segmentation fault (11) +[b20n05:121821] Signal code: (3) +[b20n05:121821] Failing at address: 0xb170000000000a0 +[b20n05:121821] [ 0] [b02n12:180113] [ 0] [0x1000000504d8] +[b02n12:180113] [ 1] [0x3fffe2c31310] +[b02n12:180113] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n12:180113] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n12:180113] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n12:180113] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n12:180113] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n12:180113] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n12:180113] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n07:173043] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n07:173043] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n07:173043] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n07:173043] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n07:173043] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n07:173043] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n07:173043] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n07:173043] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n07:173043] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n07:173043] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n07:173043] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n07:173043] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n07:173043] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n18:122556] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n18:122556] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n07:149475] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n07:149475] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n07:149475] [ 5] [b19n16:130376] *** Process received signal *** +[b19n16:130376] Signal: Segmentation fault (11) +[b19n16:130376] Signal code: (3) +[b19n16:130376] Failing at address: 0x14170000000000a0 +[b19n16:130376] [ 0] [0x1000000504d8] +[b19n16:130376] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n16:147286] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n16:147286] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n16:147286] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n16:147286] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n18:03311] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n18:03311] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n18:03311] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n18:03311] [10] [b02n04:168489] *** Process received signal *** +[b02n04:168489] Signal: Segmentation fault (11) +[b02n04:168489] Signal code: (3) +[b02n04:168489] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n10:126193] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n10:126193] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n10:126193] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n10:126193] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n14:120639] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n17:125809] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n17:125809] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n17:125809] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n17:125809] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n13:138091] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n13:138091] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n15:32401] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n15:32401] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n10:141233] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n10:141233] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n10:141233] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n10:141233] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n10:141233] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n15:148166] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n15:148166] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n15:148166] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n10:124919] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n10:124919] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n10:124919] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n10:124919] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n10:124919] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n10:124919] [10] [a32n12:164383] *** Process received signal *** +[a32n12:164383] Signal: Segmentation fault (11) +[a32n12:164383] Signal code: (3) +[a32n12:164383] Failing at address: 0x14170000000000a0 +[a32n12:164383] [ 0] [0x1000000504d8] +[a32n12:164383] [ 1] [0x3fffd3088e90] +[a32n12:164383] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n12:164383] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n14:136135] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n14:136135] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n14:136135] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n14:136135] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n14:136135] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n11:124446] [ 5] [b22n15:120061] *** Process received signal *** +[b09n01:163023] *** Process received signal *** +[b09n01:163023] Signal: Segmentation fault (11) +[b09n01:163023] Signal code: (3) +[b09n01:163023] Failing at address: 0x14170000000000a0 +[b09n01:163023] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n15:119255] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n15:119255] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n15:119255] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n15:119255] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n14:150134] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n14:150134] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n14:119488] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n14:119488] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n14:119488] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n09:173060] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n09:173060] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n09:173060] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n15:151251] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n15:151251] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n15:151251] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n15:151251] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n15:151251] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n18:176808] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n18:176808] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n18:176808] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n18:176808] [14] [b21n13:117508] *** Process received signal *** +[b21n13:117508] Signal: Segmentation fault (11) +[b21n13:117508] Signal code: (3) +[b21n13:117508] Failing at address: 0x14170000000000a0 +[b21n13:117508] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n03:158656] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n03:158656] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n03:158656] [11] [0x1000000504d8] +[a32n04:153963] [ 1] [0x3ffffc9e59d0] +[a32n04:153963] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n04:153963] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n04:153963] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n04:153963] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n04:153963] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n06:164014] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n06:164014] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n06:164014] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n17:168159] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n17:168159] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n17:168159] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n17:168159] [12] [b06n13:166924] *** Process received signal *** +[b06n13:166924] Signal: Segmentation fault (11) +[b06n13:166924] Signal code: (3) +[b06n13:166924] Failing at address: 0x14170000000000a0 +[b06n13:166924] [ 0] [0x1000000504d8] +[b06n13:166924] [ 1] [0x3fffc1a256f0] +[b06n13:166924] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n13:166924] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n12:178717] [ 5] [a30n11:163261] *** Process received signal *** +[a30n11:163261] Signal: Segmentation fault (11) +[a30n11:163261] Signal code: (3) +[a30n11:163261] Failing at address: 0x14170000000000a0 +[a30n11:163261] [ 0] [0x1000000504d8] +[a30n11:163261] [ 1] [0x3ffffeb37270] +[a30n11:163261] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n11:163261] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n04:126344] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n04:126344] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n04:126344] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n04:126344] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n04:126344] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n04:126344] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n02:119533] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n02:119533] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n02:119533] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n02:119533] [10] [b13n10:134734] *** Process received signal *** +[b13n10:134734] Signal: Segmentation fault (11) +[b13n10:134734] Signal code: (3) +[b13n10:134734] Failing at address: 0xb170000000000a0 +[b13n10:134734] [ 0] [0x1000000504d8] +[b13n10:134734] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n16:173550] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n16:173550] [ 5] [a29n11:162469] [ 0] [0x1000000504d8] +[a29n11:162469] [ 1] [0x3fffc58eb570] +[a29n11:162469] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n11:162469] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n05:125613] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n05:125613] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n02:177547] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n18:164796] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n18:164796] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n18:164796] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n18:164796] [14] [b16n11:124432] *** Process received signal *** +[b16n11:124432] Signal: Segmentation fault (11) +[b16n11:124432] Signal code: (3) +[b16n11:124432] Failing at address: 0x5170000000000a0 +[b16n11:124432] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n18:131584] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n13:168161] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n13:168161] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n13:168161] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n16:149062] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n18:08428] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n18:08428] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n18:08428] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n18:08428] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n18:08428] [13] [b01n10:173679] *** Process received signal *** +[b01n10:173679] Signal: Segmentation fault (11) +[b01n10:173679] Signal code: (3) +[b01n10:173679] Failing at address: 0xb170000000000a0 +[b01n10:173679] [ 0] [0x1000000504d8] +[b01n10:173679] [ 1] [0x3ffffff66530] +[b01n10:173679] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n10:173679] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n10:173679] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n10:173679] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n10:173679] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n10:173679] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n10:173679] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n10:173679] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n10:173679] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n10:173679] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n10:173679] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n10:173679] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n10:173679] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n10:173679] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n17:129933] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n17:129933] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n09:161231] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n09:161231] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n09:161231] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n09:161231] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n12:120131] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n12:120131] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n15:124800] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n15:124800] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n15:124800] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n15:124800] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n15:124800] *** End of error message *** +[0x1000000504d8] +[b18n08:131379] [ 1] [0x3fffd77af200] +[b18n08:131379] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n08:131379] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n08:131379] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n13:123885] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n13:123885] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n11:153416] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n11:153416] [ 6] [b02n05:164945] *** Process received signal *** +[b02n05:164945] Signal: Segmentation fault (11) +[b02n05:164945] Signal code: (3) +[b02n05:164945] Failing at address: 0x14170000000000a0 +[b02n05:164945] [ 0] [0x1000000504d8] +[b02n05:164945] [ 1] [0x3fffd36f89b0] +[b02n05:164945] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n16:128438] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n16:128438] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n16:128438] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n16:128438] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n16:128438] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n08:54626] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n08:54626] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n10:162610] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n10:162610] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n17:171198] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n17:171198] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n14:133065] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n14:133065] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n14:133065] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n14:133065] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n14:133065] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n06:127276] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n06:127276] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n06:127276] [ 9] [b11n04:166204] *** Process received signal *** +[b11n04:166204] Signal: Segmentation fault (11) +[b11n04:166204] Signal code: (3) +[b11n04:166204] Failing at address: 0x14170000000000a0 +[b11n04:166204] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n02:167599] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n02:167599] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n02:167599] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n02:167599] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n09:128000] *** End of error message *** +[a34n18:140387] *** Process received signal *** +[a34n18:140387] Signal: Segmentation fault (11) +[a34n18:140387] Signal code: (3) +[a34n18:140387] Failing at address: 0x14170000000000a0 +[a34n18:140387] [ 0] [0x1000000504d8] +[a34n18:140387] [ 1] [0x3fffc0f29220] +[a34n18:140387] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n16:165920] *** End of error message *** +[b21n08:116174] *** Process received signal *** +[b21n08:116174] Signal: Segmentation fault (11) +[b21n08:116174] Signal code: (3) +[b21n08:116174] Failing at address: 0x14170000000000a0 +[b21n08:116174] [ 0] [0x1000000504d8] +[b21n08:116174] [ 1] [0x3fffc50d0820] +[b21n08:116174] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n08:116174] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n08:116174] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n04:52707] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n04:52707] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n04:52707] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n04:52707] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n07:117359] [14] [b18n17:124163] *** Process received signal *** +[b18n17:124163] Signal: Segmentation fault (11) +[b18n17:124163] Signal code: (3) +[b18n17:124163] Failing at address: 0xb170000000000a0 +[b18n17:124163] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n07:133372] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n07:133372] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n07:133372] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n07:133372] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n07:133372] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n02:176051] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n02:176051] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n02:176051] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n02:176051] [10] [a02n04:41671] Signal: Segmentation fault (11) +[a02n04:41671] Signal code: (3) +[a02n04:41671] Failing at address: 0x14170000000000a0 +[a02n04:41671] [ 0] [0x1000000504d8] +[a02n04:41671] [ 1] [0x3fffe9f9c8f0] +[a02n04:41671] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n04:41671] [ 3] [a02n18:21769] *** Process received signal *** +[a02n18:21769] Signal: Segmentation fault (11) +[a02n18:21769] Signal code: (3) +[a02n18:21769] Failing at address: 0xb170000000000a0 +[a02n18:21769] [ 0] [0x1000000504d8] +[a02n18:21769] [ 1] [0x3fffe56110f0] +[a02n18:21769] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n18:21769] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n18:21769] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n18:21769] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n18:21769] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n18:21769] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n18:21769] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n18:21769] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n18:21769] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n18:21769] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n18:21769] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n18:21769] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n18:21769] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n18:21769] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n12:132099] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n12:132099] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n12:132099] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n12:132099] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n12:132099] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n12:132099] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n16:127352] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n16:127352] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n16:127352] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n16:127352] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n18:07422] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n18:07422] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n18:07422] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n18:07422] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n06:167368] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n06:167368] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n06:167368] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n06:167368] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n06:167368] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n11:23061] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n14:168374] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n14:168374] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n14:168374] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n08:172899] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n08:172899] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n08:172899] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n14:176070] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n09:149099] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n09:149099] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n09:149099] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n09:149099] *** End of error message *** +[b22n13:122030] *** Process received signal *** +[b22n13:122030] Signal: Segmentation fault (11) +[b22n13:122030] Signal code: (3) +[b22n13:122030] Failing at address: 0xb170000000000a0 +[b22n13:122030] [ 0] [0x1000000504d8] +[b22n13:122030] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n11:170896] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n11:170896] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n11:170896] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n11:170896] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n11:170896] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n11:170896] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n11:170896] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n11:170896] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n11:170896] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n11:170896] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n11:170896] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n11:170896] *** End of error message *** +[b20n01:128830] *** Process received signal *** +[b20n01:128830] Signal: Segmentation fault (11) +[b20n01:128830] Signal code: (3) +[b20n01:128830] Failing at address: 0x14170000000000a0 +[b20n01:128830] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n07:180193] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n07:180193] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n07:180193] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n07:180193] [14] [0x1000000504d8] +[a02n03:38502] [ 1] [0x3fffe5d8f140] +[a02n03:38502] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n03:38502] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n03:38502] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n03:38502] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n03:38502] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n03:123615] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n03:123615] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n03:123615] [ 6] [a34n16:135789] *** Process received signal *** +[a34n16:135789] Signal: Segmentation fault (11) +[a34n16:135789] Signal code: (3) +[a34n16:135789] Failing at address: 0xb170000000000a0 +[a34n16:135789] [ 0] [0x1000000504d8] +[a34n16:135789] [ 1] [0x3fffced26550] +[a34n16:135789] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n16:135789] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n16:135789] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n16:135789] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n16:135789] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n16:135789] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n16:135789] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n16:135789] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n16:135789] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n16:135789] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n16:135789] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n16:135789] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n16:135789] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n16:135789] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n11:133970] *** End of error message *** +[a27n01:48802] *** Process received signal *** +[a27n01:48802] Signal: Segmentation fault (11) +[a27n01:48802] Signal code: (3) +[a27n01:48802] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n11:160347] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n11:160347] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n11:160347] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n11:160347] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n12:167678] [11] [b23n13:114395] *** Process received signal *** +[b23n13:114395] Signal: Segmentation fault (11) +[b23n13:114395] Signal code: (3) +[b23n13:114395] Failing at address: 0xb170000000000a0 +[b23n13:114395] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n12:12764] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n12:12764] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n12:12764] [ 8] [0x1000000504d8] +[a27n09:51333] [ 1] [0x3ffffdee6810] +[a27n09:51333] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n09:51333] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n09:51333] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n09:51333] [ 5] [a29n15:177054] *** Process received signal *** +[a29n15:177054] Signal: Segmentation fault (11) +[a29n15:177054] Signal code: (3) +[a29n15:177054] Failing at address: 0x14170000000000a0 +[a29n15:177054] [ 0] [b21n16:116522] *** Process received signal *** +[b21n16:116522] Signal: Segmentation fault (11) +[b21n16:116522] Signal code: (3) +[b21n16:116522] Failing at address: 0xb170000000000a0 +[b21n16:116522] [ 0] [b19n06:126379] [ 0] [0x1000000504d8] +[b19n06:126379] [ 1] [a36n10:142933] *** Process received signal *** +[a36n10:142933] Signal: Segmentation fault (11) +[a36n10:142933] Signal code: (3) +[a36n10:142933] Failing at address: 0x14170000000000a0 +[a36n10:142933] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n17:149907] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n17:149907] [ 6] [a29n16:157885] *** Process received signal *** +[a29n16:157885] Signal: Segmentation fault (11) +[a29n16:157885] Signal code: (3) +[a29n16:157885] Failing at address: 0xb170000000000a0 +[a31n15:155430] *** Process received signal *** +[a31n15:155430] Signal: Segmentation fault (11) +[a31n15:155430] Signal code: (3) +[a31n15:155430] Failing at address: 0x14170000000000a0 +[a31n15:155430] [ 0] [0x1000000504d8] +[a31n15:155430] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n09:123418] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n09:123418] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n09:123418] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n09:123418] [10] [0x1000000504d8] +[b03n02:06562] [ 1] [0x3fffeba7b150] +[b03n02:06562] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n02:06562] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n02:06562] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n05:135166] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n05:135166] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n05:135166] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n05:135166] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n05:135166] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n08:118581] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n08:118581] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n08:118581] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n08:118581] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n08:118581] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n08:118581] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n08:118581] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n08:118581] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n08:118581] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n08:118581] *** End of error message *** +[b23n07:118163] *** Process received signal *** +[b23n07:118163] Signal: Segmentation fault (11) +[b23n07:118163] Signal code: (3) +[b23n07:118163] Failing at address: 0xb170000000000a0 +[b01n18:179700] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n14:128871] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n14:128871] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n14:128871] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n14:128871] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n14:128871] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n14:128871] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n17:03843] [ 5] [b14n13:122143] *** Process received signal *** +[b14n13:122143] Signal: Segmentation fault (11) +[b14n13:122143] Signal code: (3) +[b14n13:122143] Failing at address: 0xb170000000000a0 +[b14n13:122143] [ 0] [0x1000000504d8] +[b14n13:122143] [ 1] [b16n14:133635] *** Process received signal *** +[b16n14:133635] Signal: Segmentation fault (11) +[b16n14:133635] Signal code: (3) +[b16n14:133635] Failing at address: 0x5170000000000a0 +[b16n14:133635] [ 0] [0x1000000504d8] +[b16n14:133635] [ 1] [0x3fffc5a01400] +[b16n14:133635] [ 2] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n12:122963] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n12:122963] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n01:167298] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n01:167298] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n01:167298] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n01:167298] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n13:176366] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n13:176366] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n13:176366] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n13:176366] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n13:176366] [14] [0x1000000504d8] +[b02n14:169521] [ 1] [0x3fffe75ba180] +[b02n14:169521] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n14:169521] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n17:15873] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n17:15873] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n17:15873] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n17:15873] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n17:15873] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n09:03926] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n09:03926] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n09:03926] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n09:03926] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n09:03926] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n09:03926] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n09:03926] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n09:03926] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n09:03926] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n01:141818] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n01:141818] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n01:141818] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n01:141818] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n01:141818] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n01:141818] [12] [b03n06:01840] *** Process received signal *** +[b03n06:01840] Signal: Segmentation fault (11) +[b03n06:01840] Signal code: (3) +[b03n06:01840] Failing at address: 0x14170000000000a0 +[b03n06:01840] [ 0] [0x1000000504d8] +[b03n06:01840] [ 1] [0x3fffcbc72220] +[b03n06:01840] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n17:138142] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n17:138142] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n17:138142] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n17:138142] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n10:176310] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n10:176310] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n10:176310] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n03:177925] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n03:177925] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n03:177925] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n07:138202] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n07:138202] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n07:138202] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n07:138202] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n07:138202] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n08:08956] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n08:08956] [ 7] [0x1000000504d8] +[a36n06:143009] [ 1] [0x3fffdbbcbaa0] +[a36n06:143009] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n06:143009] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n06:143009] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n14:159238] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n14:159238] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n14:159238] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n14:136662] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n06:43018] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n06:43018] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n06:43018] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n06:43018] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n18:126903] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n18:126903] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n18:126903] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n18:126903] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n18:126903] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n02:166746] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n02:166746] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n02:166746] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n04:169912] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n04:169912] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n04:169912] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n04:169912] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n01:128157] *** End of error message *** +[b20n07:132086] *** Process received signal *** +[b20n07:132086] Signal: Segmentation fault (11) +[b20n07:132086] Signal code: (3) +[b20n07:132086] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n10:67857] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n02:164488] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n02:164488] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n02:164488] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n02:164488] [10] [a36n07:153174] *** Process received signal *** +[a36n07:153174] Signal: Segmentation fault (11) +[a36n07:153174] Signal code: (3) +[a36n07:153174] Failing at address: 0xb170000000000a0 +[b19n07:131673] *** Process received signal *** +[b19n07:131673] Signal: Segmentation fault (11) +[b19n07:131673] Signal code: (3) +[b19n07:131673] Failing at address: 0x14170000000000a0 +[b19n07:131673] [ 0] [0x1000000504d8] +[b19n07:131673] [ 1] [0x3ffffd7d5db0] +[b19n07:131673] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n07:131673] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n02:163028] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n02:163028] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n02:163028] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n02:163028] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n02:163028] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n02:163028] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n02:163028] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n11:171964] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n11:171964] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n11:171964] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n11:171964] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n17:150927] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n17:150927] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n16:01573] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n13:123709] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n13:123709] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n13:123709] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n13:123709] [13] [b11n17:164297] *** Process received signal *** +[b11n17:164297] Signal: Segmentation fault (11) +[b11n17:164297] Signal code: (3) +[b11n17:164297] Failing at address: 0xb170000000000a0 +[b11n17:164297] [ 0] [0x1000000504d8] +[b11n17:164297] [ 1] [0x3fffcbf13130] +[b11n17:164297] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n17:164297] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n17:164297] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n17:164297] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n17:164297] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n17:164297] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n17:164297] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n17:164297] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n17:164297] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n17:164297] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n17:164297] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n17:164297] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n17:164297] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n17:164297] *** End of error message *** +[a28n03:10289] *** Process received signal *** +[a28n03:10289] Signal: Segmentation fault (11) +[a28n03:10289] Signal code: (3) +[a28n03:10289] Failing at address: 0xb170000000000a0 +[a28n03:10289] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n08:147823] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n08:147823] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n08:147823] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n08:147823] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n08:147823] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n09:121754] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n17:156182] *** End of error message *** +[0x3ffff64299c0] +[b11n11:152985] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n11:152985] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n11:152985] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n11:137872] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n11:137872] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n11:137872] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n11:137872] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n11:137872] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n11:152842] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n11:152842] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n11:152842] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n11:152842] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n11:152842] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n18:28217] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n18:28217] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n18:28217] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n18:28217] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n18:28217] [10] [0x1000000504d8] +[b03n15:85943] [ 1] [0x3fffe0b26cc0] +[b03n15:85943] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n15:85943] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n15:85943] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n15:85943] [ 5] [0x3fffd58347a0] +[a28n11:01172] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n11:01172] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n11:01172] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n11:01172] [ 5] [b20n08:124723] *** Process received signal *** +[b20n08:124723] Signal: Segmentation fault (11) +[b20n08:124723] Signal code: (3) +[b20n08:124723] Failing at address: 0x14170000000000a0 +[b20n08:124723] [ 0] [0x1000000504d8] +[b20n08:124723] [ 1] [0x3fffcbee2340] +[b20n08:124723] [ 2] [b18n09:123811] *** Process received signal *** +[b18n09:123811] Signal: Segmentation fault (11) +[b18n09:123811] Signal code: (3) +[b18n09:123811] Failing at address: 0xb170000000000a0 +[b18n09:123811] [ 0] [0x1000000504d8] +[b18n09:123811] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n13:01300] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n13:01300] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n13:01300] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n13:01300] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n13:01300] [14] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n08:174408] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n08:174408] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n08:174408] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n08:174408] [ 6] [b21n01:130408] Signal: Segmentation fault (11) +[b21n01:130408] Signal code: (3) +[b21n01:130408] Failing at address: 0xb170000000000a0 +[b21n10:130579] *** Process received signal *** +[b21n10:130579] Signal: Segmentation fault (11) +[b21n10:130579] Signal code: (3) +[b21n10:130579] Failing at address: 0xb170000000000a0 +[b21n10:130579] [ 0] [0x1000000504d8] +[b21n10:130579] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n14:161618] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n14:161618] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n14:161618] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n14:161618] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n02:143966] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n02:143966] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n16:153291] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n16:153291] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n12:11144] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n12:11144] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n12:11144] [ 6] [b21n06:122162] *** Process received signal *** +[b21n06:122162] Signal: Segmentation fault (11) +[b21n06:122162] Signal code: (3) +[b21n06:122162] Failing at address: 0xb170000000000a0 +[0x1000000504d8] +[b14n09:119545] [ 1] [0x3fffdbb1fc80] +[b14n09:119545] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n09:119545] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n09:119545] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n16:122495] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n16:122495] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n16:122495] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n16:122495] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n09:173749] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n09:173749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n09:173749] [12] [b08n16:171915] [ 0] [0x1000000504d8] +[b08n16:171915] [ 1] [0x3fffcc911f90] +[b08n16:171915] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n16:171915] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n16:171915] [ 4] [a31n01:173781] *** Process received signal *** +[a31n01:173781] Signal: Segmentation fault (11) +[a31n01:173781] Signal code: (3) +[a31n01:173781] Failing at address: 0x14170000000000a0 +[a31n01:173781] [ 0] [0x1000000504d8] +[a31n01:173781] [ 1] [b21n03:127628] *** Process received signal *** +[b21n03:127628] Signal: Segmentation fault (11) +[b21n03:127628] Signal code: (3) +[b21n03:127628] Failing at address: 0x14170000000000a0 +[b21n03:127628] [ 0] [0x1000000504d8] +[b21n03:127628] [ 1] [0x3fffe2535bf0] +[b21n03:127628] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n03:127628] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n17:119582] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n17:119582] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n17:119582] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n17:119582] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n17:119582] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n12:123233] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n12:148381] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n12:148381] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n12:148381] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n12:148381] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n12:156917] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n12:156917] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n01:145734] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n01:145734] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n01:145734] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n01:145734] [10] [b21n02:120189] *** Process received signal *** +[b21n02:120189] Signal: Segmentation fault (11) +[b21n02:120189] Signal code: (3) +[b21n02:120189] Failing at address: 0xb170000000000a0 +[b21n02:120189] [ 0] [0x1000000504d8] +[b21n02:120189] [ 1] [0x3ffff6aa0060] +[b21n02:120189] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n02:120189] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n15:148579] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n15:148579] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n04:175999] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n02:163322] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n02:163322] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n02:163322] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n02:163322] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n10:122209] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n10:122209] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n04:138158] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n04:138158] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n04:138158] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n04:138158] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n17:164559] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n17:164559] *** End of error message *** +[a34n11:147713] Signal: Segmentation fault (11) +[a34n11:147713] Signal code: (3) +[a34n11:147713] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n18:163107] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n18:163107] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n18:163107] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n18:163107] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n18:163107] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n09:122068] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n14:173896] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n14:173896] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n14:173896] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n14:173896] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n07:125474] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n07:125474] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n07:125474] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n07:125474] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n07:125474] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n07:125474] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n07:125474] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n07:125474] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n08:163160] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n04:164046] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n04:164046] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n04:164046] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n08:129889] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n08:129889] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n08:129889] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n08:129889] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n03:168209] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n03:168209] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n07:127355] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n07:127355] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n07:127355] [12] [b06n01:165162] *** Process received signal *** +[b06n01:165162] Signal: Segmentation fault (11) +[b06n01:165162] Signal code: (3) +[b06n01:165162] Failing at address: 0xb170000000000a0 +[b06n01:165162] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n08:156372] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n08:156372] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n08:156372] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n08:156372] [14] [0x3ffff3184b20] +[b04n11:176298] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n11:176298] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n11:176298] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n11:176298] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n11:176298] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n11:176298] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n11:176298] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n11:176298] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n11:176298] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n11:176298] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n12:52453] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n13:165262] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n13:165262] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n13:165262] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n11:175595] *** End of error message *** +[b07n15:166705] *** Process received signal *** +[b07n15:166705] Signal: Segmentation fault (11) +[b07n15:166705] Signal code: (3) +[b07n15:166705] Failing at address: 0xb170000000000a0 +[b07n15:166705] [ 0] [0x1000000504d8] +[b07n15:166705] [ 1] [0x3fffdc9fb3f0] +[b07n15:166705] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n15:166705] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n15:166705] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n15:166705] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n15:166705] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n15:166705] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n15:166705] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n15:166705] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n15:166705] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n15:166705] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n15:166705] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n15:166705] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n15:166705] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n15:166705] *** End of error message *** +[a30n02:155383] *** Process received signal *** +[a30n02:155383] Signal: Segmentation fault (11) +[a30n02:155383] Signal code: (3) +[a30n02:155383] Failing at address: 0x14170000000000a0 +[a30n02:155383] [ 0] [0x1000000504d8] +[a30n02:155383] [ 1] [0x3fffc02c5640] +[a30n02:155383] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n02:155383] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n02:155383] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n02:155383] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n02:155383] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n02:155383] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n02:155383] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n02:155383] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n02:155383] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n02:155383] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n02:155383] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n02:155383] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n02:155383] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n02:155383] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n12:115637] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n08:163100] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n08:163100] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n08:163100] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n04:169409] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n06:177846] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n06:177846] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n06:177846] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n06:177846] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n06:177846] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n06:177846] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n06:177846] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n06:177846] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n06:177846] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n06:177846] *** End of error message *** +[0x3ffff0f8f6a0] +[b04n05:169785] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n05:169785] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n12:177120] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n12:177120] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n12:177120] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n12:177120] [ 9] [a30n15:154382] *** Process received signal *** +[a30n15:154382] Signal: Segmentation fault (11) +[a30n15:154382] Signal code: (3) +[a30n15:154382] Failing at address: 0xb170000000000a0 +[a30n15:154382] [ 0] [0x1000000504d8] +[a30n15:154382] [ 1] [0x3fffee206390] +[a30n15:154382] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n15:154382] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n05:148378] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n05:148378] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n05:148378] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n05:148378] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n05:148378] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n05:148378] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n08:121690] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n08:121690] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n08:121690] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n08:121690] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n13:163195] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n03:39684] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n03:39684] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n03:39684] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n03:39684] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n03:39684] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n03:39684] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n15:137007] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n15:137007] *** End of error message *** +[a35n17:146793] *** Process received signal *** +[a35n17:146793] Signal: Segmentation fault (11) +[a35n17:146793] Signal code: (3) +[a35n17:146793] Failing at address: 0xb170000000000a0 +[a35n17:146793] [ 0] [0x1000000504d8] +[a35n17:146793] [ 1] [0x3fffeafa6d30] +[a35n17:146793] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n17:146793] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n09:79007] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n09:79007] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n09:79007] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n09:79007] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n09:79007] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n09:79007] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n11:120242] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n11:120242] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n11:120242] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n11:120242] *** End of error message *** +[0x1000000504d8] +[a32n11:152816] [ 1] [0x3ffff5717f10] +[a32n11:152816] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n11:152816] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n11:152816] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n11:152816] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n11:152816] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n06:10226] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n06:10226] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n06:10226] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n06:10226] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n02:49105] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n02:49105] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n02:49105] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n02:49105] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n02:49105] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n01:138003] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n01:138003] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n01:138003] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n12:154341] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n12:154341] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n11:02684] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n11:02684] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n12:140120] *** End of error message *** +[0x1000000504d8] +[b10n03:168932] [ 1] [0x3fffef73d270] +[b10n03:168932] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n03:168932] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n03:168932] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n09:164363] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n09:164363] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n15:123040] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n15:123040] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n02:149226] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n02:149226] [ 9] [b15n17:126526] *** End of error message *** +[a30n14:163549] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n03:135165] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n03:135165] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n03:135165] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n03:135165] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n03:135165] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n03:135165] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n03:135165] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n12:11418] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n12:11418] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n15:161756] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n15:161756] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n15:161756] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n15:161756] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n01:153683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n01:153683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n01:153683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n01:153683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n01:153683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n17:119468] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n17:119468] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n17:119468] *** End of error message *** +[b22n16:124794] *** Process received signal *** +[b22n16:124794] Signal: Segmentation fault (11) +[b22n16:124794] Signal code: (3) +[b22n16:124794] Failing at address: 0x14170000000000a0 +[b22n16:124794] [ 0] [0x1000000504d8] +[b22n16:124794] [ 1] [0x3ffffd7a4f40] +[b22n16:124794] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n16:124794] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n03:167778] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n03:167778] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n03:167778] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n03:167778] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n03:167778] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n03:167778] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n03:167778] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n03:167778] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n03:167778] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n03:167778] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n14:21317] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n14:21317] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n14:21317] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n14:21317] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n14:21317] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n06:177003] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n06:177003] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n03:136667] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n03:136667] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n03:136667] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n03:136667] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n01:65727] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n01:65727] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n01:65727] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n01:65727] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n01:65727] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n01:65727] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n03:125241] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n03:125241] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n03:125241] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n03:125241] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n03:125241] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n06:154623] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n06:154623] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n05:162566] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n09:147791] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n09:147791] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n09:147791] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n09:147791] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n09:147791] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n09:147791] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n04:169705] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n04:169705] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n04:169705] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n04:169705] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n04:169705] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n11:148674] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n11:148674] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n11:148674] [ 9] [b01n03:01353] *** Process received signal *** +[b01n03:01353] Signal: Segmentation fault (11) +[b01n03:01353] Signal code: (3) +[b01n03:01353] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n04:118345] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n04:118345] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n04:118345] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n04:118345] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n04:118345] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n04:118345] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n04:118345] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n04:118345] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n04:118345] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n02:178473] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n02:178473] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n02:178473] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n02:178473] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n02:178473] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n02:178473] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n02:178473] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n02:127218] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n02:127218] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n02:127218] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n02:127218] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n02:127218] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n02:122555] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n02:122555] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n02:122555] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n02:122555] [ 7] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n09:163514] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n09:163514] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n09:163514] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n02:123704] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n02:123704] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n02:123704] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n02:123704] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n02:123704] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n10:161717] [ 5] [0x1000000504d8] +[b07n17:162357] [ 1] [0x3fffe37b1640] +[b07n17:162357] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n17:162357] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n17:162357] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n17:162357] [ 5] [a34n01:164964] *** Process received signal *** +[a34n01:164964] Signal: Segmentation fault (11) +[a34n01:164964] Signal code: (3) +[a34n01:164964] Failing at address: 0x14170000000000a0 +[a34n01:164964] [ 0] [0x1000000504d8] +[a34n01:164964] [ 1] [b20n06:130540] *** Process received signal *** +[b20n06:130540] Signal: Segmentation fault (11) +[b20n06:130540] Signal code: (3) +[b20n06:130540] Failing at address: 0xb170000000000a0 +[b20n06:130540] [ 0] [b06n17:163818] *** Process received signal *** +[b06n17:163818] Signal: Segmentation fault (11) +[b06n17:163818] Signal code: (3) +[b06n17:163818] Failing at address: 0xb170000000000a0 +[b06n17:163818] [ 0] [0x1000000504d8] +[b06n17:163818] [ 1] [0x3ffff9dce740] +[b06n17:163818] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n17:163818] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n06:133952] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n06:133952] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n06:133952] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n06:133952] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n06:133952] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n06:133952] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n06:133952] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n06:133952] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n06:133952] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n05:131261] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n05:131261] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n05:131261] [13] [0x1000000504d8] +[a28n04:13079] [ 1] [0x3fffd2e60bd0] +[a28n04:13079] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n04:13079] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n04:13079] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n05:135122] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n05:135122] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n05:135122] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n05:135122] *** End of error message *** +[0x3ffffc96b360] +[a32n03:151879] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n03:151879] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n03:151879] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n03:151879] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n02:147884] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n02:147884] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n02:147884] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n02:147884] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n01:179262] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n01:179262] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n01:179262] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n01:179262] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n08:172426] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n08:172426] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n08:172426] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n08:175805] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n08:175805] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n08:175805] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n08:175805] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n08:175805] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n08:175805] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n08:175805] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n08:175805] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n08:175805] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n05:54534] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n05:54534] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n05:54534] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n05:54534] [10] [0x3fffdcde8510] +[b08n01:171786] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n01:171786] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n01:171786] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n01:171786] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n17:118574] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n17:118574] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n17:118574] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n17:118574] [ 9] [a36n04:142255] *** Process received signal *** +[a36n04:142255] Signal: Segmentation fault (11) +[a36n04:142255] Signal code: (3) +[a36n04:142255] Failing at address: 0xb170000000000a0 +[a36n04:142255] [ 0] [0x1000000504d8] +[a36n04:142255] [ 1] [0x3ffff063ca10] +[a36n04:142255] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n04:142255] [ 3] [a36n03:153850] *** Process received signal *** +[a36n03:153850] Signal: Segmentation fault (11) +[a36n03:153850] Signal code: (3) +[a36n03:153850] Failing at address: 0xb170000000000a0 +[a36n03:153850] [ 0] [0x1000000504d8] +[a36n03:153850] [ 1] [0x3fffcbd34790] +[a36n03:153850] [ 2] [a32n05:154188] *** Process received signal *** +[a32n05:154188] Signal: Segmentation fault (11) +[a32n05:154188] Signal code: (3) +[a32n05:154188] Failing at address: 0x5170000000000a0 +[a32n05:154188] [ 0] [0x1000000504d8] +[a32n05:154188] [ 1] [0x3fffc77a02d0] +[a32n05:154188] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n05:154188] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n05:130804] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n05:130804] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n05:130804] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n05:130804] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n05:130804] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n05:130804] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n05:130804] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n05:130804] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n18:118680] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n18:118680] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n18:118680] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n18:118680] *** End of error message *** +[0x3fffc5a489b0] +[b08n11:164993] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n11:164993] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n11:164993] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n11:164993] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n01:141340] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n01:141340] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n01:141340] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n01:141340] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n01:141340] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n02:121682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n02:121682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n02:121682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n02:121682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n02:121682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n02:121682] *** End of error message *** +[b15n16:121930] *** Process received signal *** +[b15n16:121930] Signal: Segmentation fault (11) +[b15n16:121930] Signal code: (3) +[b15n16:121930] Failing at address: 0x14170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n10:164939] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n10:164939] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n10:164939] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n10:164939] *** End of error message *** +[b10n06:162915] *** Process received signal *** +[0x3fffc5910040] +[a36n14:152037] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n14:152037] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n14:152037] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n14:152037] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n14:152037] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n14:152037] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n14:152037] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n14:152037] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n04:123055] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n04:123055] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n04:123055] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n04:123055] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n04:123055] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n04:123055] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n04:123055] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n04:123055] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n04:123055] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n08:164752] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n08:164752] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n08:164752] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n01:132062] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n01:132062] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n01:132062] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n04:04849] *** End of error message *** +[b16n06:132107] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n03:130346] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n06:171503] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n06:171503] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n06:171503] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n02:116887] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n02:116887] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n02:116887] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n02:116887] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n02:116887] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n02:116887] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n02:116887] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n02:116887] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n02:116887] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n02:116887] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n02:116887] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n06:120734] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n06:120734] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n06:120734] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n06:120734] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n07:03371] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n07:03371] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n07:03371] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n07:03371] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n10:02711] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n02:139024] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n02:139024] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n02:139024] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n02:139024] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n05:173080] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n05:173080] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n05:173080] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n11:160799] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n11:160799] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n11:160799] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n11:160799] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n11:160799] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n11:160799] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n06:163678] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n06:163678] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n06:163678] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n05:120332] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n05:120332] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n05:120332] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n05:120332] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n05:120332] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n01:128005] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n01:128005] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n01:128005] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n01:128005] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n12:138738] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n16:164549] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n16:164549] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n16:164549] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n16:164549] [13] [b14n04:126858] *** Process received signal *** +[b14n04:126858] Signal: Segmentation fault (11) +[b14n04:126858] Signal code: (3) +[b14n04:126858] Failing at address: 0x14170000000000a0 +[b14n04:126858] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n17:123230] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n17:123230] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n17:123230] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n05:126782] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n07:124352] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n07:124352] *** End of error message *** +[0x1000000504d8] +[b05n13:10195] [ 1] [0x3fffd9bd8f30] +[b05n13:10195] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n13:10195] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n13:10195] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n13:10195] [ 5] [a32n09:155760] *** Process received signal *** +[a32n09:155760] Signal: Segmentation fault (11) +[a32n09:155760] Signal code: (3) +[a32n09:155760] Failing at address: 0x14170000000000a0 +[a32n09:155760] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n01:165125] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n17:176413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n17:176413] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n17:176413] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n17:176413] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n06:157789] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n06:157789] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n06:157789] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n13:153677] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n13:153677] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n13:153677] [ 6] [a35n06:138762] *** Process received signal *** +[a35n06:138762] Signal: Segmentation fault (11) +[a35n06:138762] Signal code: (3) +[a35n06:138762] Failing at address: 0xb170000000000a0 +[a35n06:138762] [ 0] [0x1000000504d8] +[a35n06:138762] [ 1] [0x3fffc4a5a700] +[a35n06:138762] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n06:138762] [ 3] [b01n12:178600] *** Process received signal *** +[b01n12:178600] Signal: Segmentation fault (11) +[b01n12:178600] Signal code: (3) +[b01n12:178600] Failing at address: 0x14170000000000a0 +[b01n12:178600] [ 0] [0x1000000504d8] +[b01n12:178600] [ 1] [0x3fffde6ce660] +[b01n12:178600] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n12:178600] [ 3] [b23n11:117473] *** Process received signal *** +[b23n11:117473] Signal: Segmentation fault (11) +[b23n11:117473] Signal code: (3) +[b23n11:117473] Failing at address: 0xb170000000000a0 +[b23n11:117473] [ 0] [0x1000000504d8] +[b23n11:117473] [ 1] [b08n06:161724] *** Process received signal *** +[b08n06:161724] Signal: Segmentation fault (11) +[b08n06:161724] Signal code: (3) +[b08n06:161724] Failing at address: 0x14170000000000a0 +[b08n06:161724] [ 0] [0x1000000504d8] +[b08n06:161724] [ 1] [0x3fffcbce9170] +[b08n06:161724] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n06:161724] [ 3] [0x3ffffa0aff50] +[a33n17:168155] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n17:168155] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n17:168155] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n07:152084] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n07:152084] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n07:152084] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n07:152084] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n07:152084] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n06:117526] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n06:117526] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n06:117526] [13] [b07n10:16974] *** Process received signal *** +[b07n10:16974] Signal: Segmentation fault (11) +[b07n10:16974] Signal code: (3) +[b07n10:16974] Failing at address: 0x14170000000000a0 +[b07n10:16974] [ 0] [0x1000000504d8] +[b07n10:16974] [ 1] [0x3fffdc18b960] +[b07n10:16974] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n10:16974] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n11:03986] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n11:03986] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n11:03986] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n11:03986] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n03:120668] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n03:120668] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n03:120668] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n03:120668] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n03:120668] *** End of error message *** +[a29n07:160800] *** Process received signal *** +[a29n07:160800] Signal: Segmentation fault (11) +[a29n07:160800] Signal code: (3) +[a29n07:160800] Failing at address: 0x14170000000000a0 +[a29n07:160800] [ 0] [0x1000000504d8] +[a29n07:160800] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n11:120581] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n11:120581] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n11:120581] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n11:120581] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n11:120581] *** End of error message *** +[a27n03:46691] [ 0] [0x1000000504d8] +[a27n03:46691] [ 1] [0x3ffff67b8a10] +[a27n03:46691] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n03:46691] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n03:46691] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n03:46691] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n03:46691] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n03:46691] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n03:46691] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n03:46691] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n03:46691] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n03:46691] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n03:46691] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n03:154294] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n03:154294] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n03:154294] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n03:154294] [13] [a31n04:158632] *** Process received signal *** +[a31n04:158632] Signal: Segmentation fault (11) +[a31n04:158632] Signal code: (3) +[a31n04:158632] Failing at address: 0x14170000000000a0 +[a31n04:158632] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n14:178326] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n14:178326] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n09:119635] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n09:119635] *** End of error message *** +[b02n13:171107] *** Process received signal *** +[b02n13:171107] Signal: Segmentation fault (11) +[b02n13:171107] Signal code: (3) +[b02n13:171107] Failing at address: 0xb170000000000a0 +[b02n13:171107] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n04:145749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n04:145749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n04:145749] [ 9] [0x3fffca942b70] +[b22n05:123936] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n05:123936] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n05:123936] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n05:123936] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n05:123936] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n05:123936] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n05:123936] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n05:123936] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n05:123936] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n05:123936] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n05:123936] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n05:123936] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n05:123936] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n05:123936] *** End of error message *** +[a33n05:139436] *** Process received signal *** +[a33n05:139436] Signal: Segmentation fault (11) +[a33n05:139436] Signal code: (3) +[a33n05:139436] Failing at address: 0x14170000000000a0 +[a33n05:139436] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n18:179169] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n18:179169] [ 5] [b17n13:123514] *** Process received signal *** +[b17n13:123514] Signal: Segmentation fault (11) +[b17n13:123514] Signal code: (3) +[b17n13:123514] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n07:168962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n07:168962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n07:168962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n07:168962] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n07:168962] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n07:168962] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n07:168962] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n11:134519] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n11:134519] [ 5] [b08n10:11492] *** Process received signal *** +[b08n10:11492] Signal: Segmentation fault (11) +[b08n10:11492] Signal code: (3) +[b08n10:11492] Failing at address: 0x14170000000000a0 +[b08n10:11492] [ 0] [0x1000000504d8] +[b08n10:11492] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n17:169734] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n17:169734] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n17:169734] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n17:169734] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n17:169734] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n04:44756] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n04:44756] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n04:44756] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n04:44756] [14] [b14n12:123561] *** Process received signal *** +[b14n12:123561] Signal: Segmentation fault (11) +[b14n12:123561] Signal code: (3) +[b14n12:123561] Failing at address: 0xb170000000000a0 +[a36n05:139912] [ 0] [0x1000000504d8] +[a36n05:139912] [ 1] [0x3fffeb1e1ee0] +[a36n05:139912] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n05:139912] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n05:139912] [ 4] [b03n03:06823] *** Process received signal *** +[b03n03:06823] Signal: Segmentation fault (11) +[b03n03:06823] Signal code: (3) +[b03n03:06823] Failing at address: 0x14170000000000a0 +[0x3fffc15cea30] +[a30n05:156007] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n05:156007] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n05:156007] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n05:156007] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n05:156007] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n05:156007] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n05:156007] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n05:156007] [ 9] [a30n10:169210] *** Process received signal *** +[a30n10:169210] Signal: Segmentation fault (11) +[a30n10:169210] Signal code: (3) +[a30n10:169210] Failing at address: 0x14170000000000a0 +[a30n10:169210] [ 0] [0x1000000504d8] +[a30n10:169210] [ 1] [b15n14:133970] Signal: Segmentation fault (11) +[b15n14:133970] Signal code: (3) +[b15n14:133970] Failing at address: 0xb170000000000a0 +[b15n14:133970] [ 0] [0x1000000504d8] +[b15n14:133970] [ 1] [0x3fffee0e5590] +[b15n14:133970] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n14:133970] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n14:133970] [ 4] [a29n05:174780] *** Process received signal *** +[a29n05:174780] Signal: Segmentation fault (11) +[a29n05:174780] Signal code: (3) +[a29n05:174780] Failing at address: 0xb170000000000a0 +[a29n05:174780] [ 0] [b18n13:123051] *** Process received signal *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n14:08399] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n14:08399] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n13:154231] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n13:154231] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n13:154231] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n01:163391] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n01:163391] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n01:163391] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n01:163391] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n01:163391] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n01:163391] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n01:163391] [14] [0x1000000504d8] +[b22n10:118390] [ 1] [0x3fffe51b12d0] +[b22n10:118390] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n10:118390] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n10:118390] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n08:141874] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n08:141874] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n08:141874] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n08:141874] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n08:141874] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n04:03148] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n04:03148] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n04:03148] [ 6] [b07n12:160963] *** Process received signal *** +[b07n12:160963] Signal: Segmentation fault (11) +[b07n12:160963] Signal code: (3) +[b07n12:160963] Failing at address: 0x14170000000000a0 +[b07n12:160963] [ 0] [0x1000000504d8] +[b07n12:160963] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n08:176680] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n08:176680] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n08:176680] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n08:176680] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n08:176680] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n12:145497] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n12:145497] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n12:145497] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n12:145497] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n12:145497] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n01:96754] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n01:96754] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n01:96754] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n01:96754] [10] [0x3ffff8f7a690] +[b04n09:12246] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n09:12246] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n09:12246] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n09:12246] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n09:12246] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n09:12246] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n09:12246] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n09:12246] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n15:166119] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n11:130803] [ 4] [b23n05:120227] *** Process received signal *** +[b03n01:170766] *** Process received signal *** +[b03n01:170766] Signal: Segmentation fault (11) +[b03n01:170766] Signal code: (3) +[b03n01:170766] Failing at address: 0x14170000000000a0 +[b03n01:170766] [ 0] [0x1000000504d8] +[b03n01:170766] [ 1] [0x3fffd545e7d0] +[b03n01:170766] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n01:170766] [ 3] [b13n18:128543] *** Process received signal *** +[b13n18:128543] Signal: Segmentation fault (11) +[b13n18:128543] Signal code: (3) +[b13n18:128543] Failing at address: 0xb170000000000a0 +[b13n18:128543] [ 0] [0x1000000504d8] +[b13n18:128543] [ 1] [0x3ffff103d080] +[b13n18:128543] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n18:128543] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n06:179453] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n06:179453] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n06:179453] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n06:179453] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n06:179453] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n06:179453] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n10:140416] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n10:140416] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n10:140416] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n10:140416] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n10:140416] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n10:140416] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n10:140416] *** End of error message *** +[a29n08:159024] *** Process received signal *** +[a29n08:159024] Signal: Segmentation fault (11) +[a29n08:159024] Signal code: (3) +[a29n08:159024] Failing at address: 0x14170000000000a0 +[a29n08:159024] [ 0] [0x1000000504d8] +[a29n08:159024] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n16:154183] [ 5] [b07n03:172649] *** Process received signal *** +[b07n03:172649] Signal: Segmentation fault (11) +[b07n03:172649] Signal code: (3) +[b07n03:172649] Failing at address: 0x14170000000000a0 +[b07n03:172649] [ 0] [0x1000000504d8] +[b07n03:172649] [ 1] [0x3fffcf521cd0] +[b07n03:172649] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n15:120132] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n13:133624] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n10:126611] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n10:126611] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n01:124278] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n01:124278] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n01:124278] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n01:124278] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n11:156155] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n11:156155] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n11:156155] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n11:156155] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n11:156155] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n11:156155] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n08:176174] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n08:176174] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n08:176174] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n08:176174] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n08:176174] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n08:176174] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n08:176174] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n06:119114] *** End of error message *** +[0x3fffc0822260] +[b20n02:122181] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n02:122181] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n02:122181] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n15:139746] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n15:139746] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n15:139746] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n15:139746] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n15:139746] *** End of error message *** +[a32n17:138612] *** Process received signal *** +[a32n17:138612] Signal: Segmentation fault (11) +[a32n17:138612] Signal code: (3) +[a32n17:138612] Failing at address: 0x5170000000000a0 +[a32n17:138612] [ 0] [0x1000000504d8] +[a32n17:138612] [ 1] [0x3fffe958ad70] +[a32n17:138612] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n17:138612] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n17:138612] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n17:138612] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n17:138612] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n17:138612] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n17:138612] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n17:138612] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n17:138612] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n17:138612] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n17:138612] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n17:138612] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n17:138612] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n17:138612] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n05:04521] *** End of error message *** +[0x1000000504d8] +[b15n15:122085] [ 1] [0x3fffe6cc7c50] +[b15n15:122085] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n15:122085] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n15:122085] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n15:122085] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n07:165092] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n07:165092] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n07:165092] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n07:165092] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n07:165092] [10] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n08:127445] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n08:127445] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n08:127445] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n08:127445] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n14:126465] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n14:126465] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n14:126465] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n14:126465] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n14:126465] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n09:168245] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n09:168245] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n09:168245] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n09:168245] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n14:152747] [ 5] [b05n10:10926] *** Process received signal *** +[b05n10:10926] Signal: Segmentation fault (11) +[b05n10:10926] Signal code: (3) +[b05n10:10926] Failing at address: 0x14170000000000a0 +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n05:173316] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n05:173316] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n09:160372] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n09:160372] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n09:160372] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n09:160372] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n09:160372] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n09:160372] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n09:160372] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n09:160372] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n09:160372] *** End of error message *** +[a31n10:155400] *** Process received signal *** +[a31n10:155400] Signal: Segmentation fault (11) +[a31n10:155400] Signal code: (3) +[a31n10:155400] Failing at address: 0x14170000000000a0 +[a31n10:155400] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n01:121743] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n01:121743] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n01:121743] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n01:121743] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n01:121743] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n01:121743] *** End of error message *** +[a02n10:135851] *** Process received signal *** +[a02n10:135851] Signal: Segmentation fault (11) +[a02n10:135851] Signal code: (3) +[a02n10:135851] Failing at address: 0xb170000000000a0 +[a02n10:135851] [ 0] [0x1000000504d8] +[a02n10:135851] [ 1] [0x3fffee066850] +[a02n10:135851] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n10:135851] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n10:169683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n10:169683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n10:169683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n10:169683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n10:169683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n10:169683] *** End of error message *** +[a26n02:74431] *** Process received signal *** +[a26n02:74431] Signal: Segmentation fault (11) +[a26n02:74431] Signal code: (3) +[a26n02:74431] Failing at address: 0x14170000000000a0 +[a26n02:74431] [ 0] [0x1000000504d8] +[a26n02:74431] [ 1] [0x3fffe4c264b0] +[a26n02:74431] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n02:74431] [ 3] [a32n07:172879] *** Process received signal *** +[a32n07:172879] Signal: Segmentation fault (11) +[a32n07:172879] Signal code: (3) +[a32n07:172879] Failing at address: 0x14170000000000a0 +[a32n07:172879] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n10:120977] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n10:120977] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n10:120977] [ 5] [b06n03:164495] Signal: Segmentation fault (11) +[b06n03:164495] Signal code: (3) +[b06n03:164495] Failing at address: 0x14170000000000a0 +[b06n03:164495] [ 0] [0x1000000504d8] +[b06n03:164495] [ 1] [0x3ffff89bae50] +[b06n03:164495] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n03:164495] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n03:164495] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n04:130748] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n04:130748] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n10:122756] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n10:122756] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n12:122875] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n12:122875] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n13:155850] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n13:155850] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n13:155850] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n05:149465] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n05:149465] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n05:149465] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n05:149465] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n05:149465] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n02:174057] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n02:174057] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n02:174057] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n02:174057] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n02:174057] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n02:174057] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n02:174057] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n02:174057] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n02:174057] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n02:174057] *** End of error message *** +[a27n07:45308] *** Process received signal *** +[a27n07:45308] Signal: Segmentation fault (11) +[a27n07:45308] Signal code: (3) +[a27n07:45308] Failing at address: 0x14170000000000a0 +[a27n07:45308] [ 0] [0x1000000504d8] +[a27n07:45308] [ 1] [0x3fffc628c180] +[a27n07:45308] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n07:45308] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n09:106786] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n09:106786] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n09:106786] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n09:106786] [10] [a29n12:160190] [ 0] [0x1000000504d8] +[a29n12:160190] [ 1] [0x3fffc826db70] +[a29n12:160190] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n12:160190] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n14:177513] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n14:177513] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n14:177513] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n14:177513] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n14:177513] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n14:177513] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n14:177513] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n14:177513] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n14:177513] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n14:177513] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n14:177513] *** End of error message *** +[b18n02:125374] *** Process received signal *** +[b18n02:125374] Signal: Segmentation fault (11) +[b18n02:125374] Signal code: (3) +[b18n02:125374] Failing at address: 0x14170000000000a0 +[b18n02:125374] [ 0] [0x1000000504d8] +[b18n02:125374] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n05:176911] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n05:176911] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n05:176911] [10] [b09n09:170656] *** Process received signal *** +[b09n09:170656] Signal: Segmentation fault (11) +[b09n09:170656] Signal code: (3) +[b09n09:170656] Failing at address: 0xb170000000000a0 +[b09n09:170656] [ 0] [0x1000000504d8] +[b09n09:170656] [ 1] [0x3fffff041370] +[b09n09:170656] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n16:124960] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n16:124960] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n16:124960] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n04:156413] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n04:156413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n04:156413] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n04:156413] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n08:146116] [ 5] [a31n07:156490] *** Process received signal *** +[a31n07:156490] Signal: Segmentation fault (11) +[a31n07:156490] Signal code: (3) +[a31n07:156490] Failing at address: 0x14170000000000a0 +[a31n07:156490] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n03:135037] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n03:135037] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n03:135037] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n09:128493] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n09:128493] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n09:128493] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n09:128493] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n09:128493] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n09:128493] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n09:128493] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n17:163177] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n17:163177] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n17:163177] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n07:02840] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n07:02840] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n07:02840] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n07:02840] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n07:02840] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n14:126119] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n14:126119] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n14:126119] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n14:126119] [ 9] [0x1000000504d8] +[b06n10:176263] [ 1] [0x3ffffd868450] +[b06n10:176263] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n10:176263] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n10:176263] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n10:176263] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n10:176263] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n04:136996] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n04:136996] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n04:136996] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n04:136996] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n04:136996] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n03:126187] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n03:126187] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n03:126187] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n03:126187] [10] [b15n08:118551] *** Process received signal *** +[b15n08:118551] Signal: Segmentation fault (11) +[b15n08:118551] Signal code: (3) +[b15n08:118551] Failing at address: 0xb170000000000a0 +[b15n08:118551] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n15:165474] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n15:165474] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n15:165474] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n15:165474] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n15:165474] *** End of error message *** +[a33n03:147534] *** Process received signal *** +[a33n03:147534] Signal: Segmentation fault (11) +[a33n03:147534] Signal code: (3) +[a33n03:147534] Failing at address: 0x5170000000000a0 +[a33n03:147534] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n11:134749] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n11:134749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n11:134749] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n03:68832] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n03:68832] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n03:68832] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n03:68832] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n03:68832] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n03:68832] *** End of error message *** +[0x1000000504d8] +[a34n13:137748] [ 1] [0x3fffdb281d40] +[a34n13:137748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n13:137748] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n13:137748] [ 4] [a35n12:145265] [ 0] [0x1000000504d8] +[a35n12:145265] [ 1] [0x3ffff9ee91a0] +[a35n12:145265] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n12:145265] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n08:165281] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n08:165281] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n14:122443] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n14:122443] *** End of error message *** +[0x3ffff7dc7e50] +[b15n01:123570] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n01:123570] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n01:123570] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n05:173822] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n05:173822] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n05:173822] *** End of error message *** +[0x1000000504d8] +[b15n03:119831] [ 1] [0x3ffff018ea20] +[b15n03:119831] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n03:119831] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n03:119831] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n07:125451] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n07:125451] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n07:125451] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n07:125451] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n07:125451] [11] [b17n08:124114] *** Process received signal *** +[b17n08:124114] Signal: Segmentation fault (11) +[b17n08:124114] Signal code: (3) +[b17n08:124114] Failing at address: 0x14170000000000a0 +[b17n08:124114] [ 0] [0x1000000504d8] +[b17n08:124114] [ 1] [0x3ffffe62a560] +[b17n08:124114] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n13:165387] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n13:165387] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n13:165387] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n13:165387] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n13:165387] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n13:165387] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n13:165387] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n13:165387] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n13:165387] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n11:128953] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n11:128953] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n11:128953] [11] [0x3fffc3ecd1e0] +[a34n06:148311] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n06:148311] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n06:148311] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n04:136609] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n04:136609] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n04:136609] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n03:124681] [ 5] [b07n07:163252] *** Process received signal *** +[b07n07:163252] Signal: Segmentation fault (11) +[b07n07:163252] Signal code: (3) +[b07n07:163252] Failing at address: 0xb170000000000a0 +[b07n07:163252] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n12:120405] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n12:120405] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n12:120405] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n12:120405] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n12:120405] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n16:129236] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n16:129236] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n07:156140] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n07:156140] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n07:156140] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n07:156140] [10] [a33n12:168643] *** Process received signal *** +[a33n12:168643] Signal: Segmentation fault (11) +[a33n12:168643] Signal code: (3) +[a33n12:168643] Failing at address: 0xb170000000000a0 +[a33n12:168643] [ 0] [0x1000000504d8] +[a33n12:168643] [ 1] [0x3ffff97d08e0] +[a33n12:168643] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n12:168643] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n12:168643] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n12:168643] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n12:168643] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n12:168643] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n12:168643] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n12:168643] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n12:168643] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n12:168643] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n12:168643] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n12:168643] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n12:168643] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n12:168643] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n14:116410] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n14:116410] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n14:116410] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n14:116410] [14] [b03n14:177222] *** Process received signal *** +[b03n14:177222] Signal: Segmentation fault (11) +[b03n14:177222] Signal code: (3) +[b03n14:177222] Failing at address: 0x5170000000000a0 +[b03n14:177222] [ 0] [0x1000000504d8] +[b03n14:177222] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n18:121306] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n18:121306] [ 5] [b03n05:05606] *** Process received signal *** +[b03n05:05606] Signal: Segmentation fault (11) +[b03n05:05606] Signal code: (3) +[b03n05:05606] Failing at address: 0xb170000000000a0 +[b03n05:05606] [ 0] [0x1000000504d8] +[b03n05:05606] [ 1] [0x3fffeb197ce0] +[b03n05:05606] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n05:05606] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n04:177598] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n04:177598] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n04:177598] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n04:177598] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n13:177637] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n13:177637] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n13:177637] [12] [0x3fffe6222360] +[b13n09:143896] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n09:143896] [ 3] [b12n14:139843] [ 0] [0x1000000504d8] +[b12n14:139843] [ 1] [0x3fffde336110] +[b12n14:139843] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n14:139843] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n14:139843] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n14:139843] [ 5] [0x1000000504d8] +[b23n04:117352] [ 1] [0x3fffe7f39e20] +[b23n04:117352] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n04:117352] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n04:117352] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n08:144855] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n08:144855] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n08:144855] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n08:144855] [ 9] [a28n12:177494] *** Process received signal *** +[a28n12:177494] Signal: Segmentation fault (11) +[a28n12:177494] Signal code: (3) +[a28n12:177494] Failing at address: 0xb170000000000a0 +[a28n12:177494] [ 0] [0x1000000504d8] +[a28n12:177494] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n05:53373] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n05:53373] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n05:53373] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n05:53373] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n09:166574] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n09:166574] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n09:166574] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n09:166574] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n09:166574] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n09:166574] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n08:138174] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n08:138174] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n08:138174] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n13:54242] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n13:54242] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n13:54242] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n13:54242] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n13:54242] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n13:54242] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n12:119806] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n12:119806] *** End of error message *** +[b20n05:121822] *** Process received signal *** +[b20n05:121822] Signal: Segmentation fault (11) +[b20n05:121822] Signal code: (3) +[b20n05:121822] Failing at address: 0xb170000000000a0 +[b20n05:121822] [ 0] [0x1000000504d8] +[b20n05:121822] [ 1] [0x1000000504d8] +[b02n12:180114] [ 1] [0x3fffc97d8260] +[b02n12:180114] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n12:180114] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n12:180114] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n12:180114] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n12:180114] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n12:180114] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n12:180114] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n12:180114] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n18:122553] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n18:122553] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n18:122553] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n18:122553] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n07:149476] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n07:149476] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n07:149476] [ 6] [0x3fffd8ce7bf0] +[b19n16:130377] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n16:130377] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n16:130377] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n16:130377] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n16:147287] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n16:147287] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n16:147287] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n16:147287] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n16:147287] [10] [b02n04:168489] [ 0] [0x1000000504d8] +[b02n04:168489] [ 1] [0x3ffff056b1b0] +[b02n04:168489] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n04:168489] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n04:168489] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n18:03312] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n18:03312] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n18:03312] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n10:126194] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n10:126194] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n10:126194] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n10:126194] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n10:126194] *** End of error message *** +[b17n17:125808] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n10:141234] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n10:141234] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n10:141234] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n10:141234] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n10:141234] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n15:148167] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n15:148167] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n15:148167] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n15:148167] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n13:138090] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n13:138090] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n13:138090] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n13:138090] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n13:138090] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n10:124918] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n10:124918] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n10:124918] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n10:124918] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n10:124918] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n10:124918] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n10:124918] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n10:124918] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n10:124918] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n15:32401] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n12:164382] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n12:164382] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n11:124446] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n11:124446] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n11:124446] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n14:136136] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n14:136136] [ 5] [0x1000000504d8] +[b09n01:163023] [ 1] [0x3fffff2cae20] +[b09n01:163023] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n01:163023] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n01:163023] [ 4] [b22n15:120061] Signal: Segmentation fault (11) +[b22n15:120061] Signal code: (3) +[b22n15:120061] Failing at address: 0x14170000000000a0 +[b22n15:120061] [ 0] [0x1000000504d8] +[b22n15:120061] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n15:119254] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n15:119254] [ 5] [a36n15:151249] *** Process received signal *** +[a34n14:150131] *** Process received signal *** +[a34n14:150131] Signal: Segmentation fault (11) +[a34n14:150131] Signal code: (3) +[a34n14:150131] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n14:119488] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n14:119488] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n18:176809] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n18:176809] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n18:176809] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n18:176809] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n18:176809] *** End of error message *** +[0x1000000504d8] +[a30n11:163260] [ 1] [0x3fffd324aaa0] +[a30n11:163260] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n11:163260] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n11:163260] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n04:153962] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n04:153962] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n04:153962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n12:178717] [ 6] [0x1000000504d8] +[b21n13:117508] [ 1] [0x3fffd526fa90] +[b21n13:117508] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n13:117508] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n13:117508] [ 4] [0x3fffd7f1a370] +[b13n10:134733] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n10:134733] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n10:134733] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n10:134733] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n17:168160] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n17:168160] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n17:168160] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n17:168160] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n04:126345] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n04:126345] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n04:126345] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n04:126345] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n04:126345] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n04:126345] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n13:166924] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n13:166924] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n06:164013] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n06:164013] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n02:119534] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n02:119534] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n02:119534] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n02:119534] [14] [a29n11:162470] *** Process received signal *** +[a29n11:162470] Signal: Segmentation fault (11) +[a29n11:162470] Signal code: (3) +[a29n11:162470] Failing at address: 0xb170000000000a0 +[a29n11:162470] [ 0] [0x1000000504d8] +[a29n11:162470] [ 1] [0x3fffdcfdfd70] +[a29n11:162470] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n11:162470] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n16:173550] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n16:173550] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n16:173550] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n16:173550] [ 9] [0x1000000504d8] +[b16n11:124432] [ 1] [0x3ffff363bac0] +[b16n11:124432] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n11:124432] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n11:124432] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n13:168162] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n03:158655] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n03:158655] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n03:158655] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n11:153415] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n11:153415] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n11:153415] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n11:153415] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n16:149063] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n16:149063] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n18:164797] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n18:164797] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n05:125612] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n05:125612] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n05:125612] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n05:125612] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n05:125612] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n09:161232] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n09:161232] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n18:08427] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n18:08427] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n18:08427] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n18:08427] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n10:173682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n10:173682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n10:173682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n10:173682] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n10:173682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n10:173682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n10:173682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n10:173682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n10:173682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n10:173682] *** End of error message *** +[b21n12:120128] *** Process received signal *** +[b21n12:120128] Signal: Segmentation fault (11) +[b21n12:120128] Signal code: (3) +[b21n12:120128] Failing at address: 0xb170000000000a0 +[b21n12:120128] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n17:129932] *** End of error message *** +[b19n15:124801] *** Process received signal *** +[b19n15:124801] Signal: Segmentation fault (11) +[b19n15:124801] Signal code: (3) +[b19n15:124801] Failing at address: 0x14170000000000a0 +[b19n15:124801] [ 0] [0x1000000504d8] +[b19n15:124801] [ 1] [0x3fffed4331f0] +[b19n15:124801] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n15:124801] [ 3] [a34n18:140386] *** Process received signal *** +[a34n18:140386] Signal: Segmentation fault (11) +[a34n18:140386] Signal code: (3) +[a34n18:140386] Failing at address: 0x14170000000000a0 +[a34n18:140386] [ 0] [0x1000000504d8] +[a34n18:140386] [ 1] [0x3fffe0ffb4b0] +[a34n18:140386] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n18:140386] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n13:123885] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n13:123885] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n13:123885] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n13:123885] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n13:123885] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n13:123885] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n13:123885] [13] [0x3fffc38e2b30] +[b02n05:164944] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n05:164944] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n05:164944] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n18:131584] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n18:131584] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n16:128439] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n16:128439] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n16:128439] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n16:128439] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n08:54626] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n08:54626] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n08:54626] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n08:54626] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n10:162611] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n17:171199] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n17:171199] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n17:171199] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n17:171199] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n14:133065] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n14:133065] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n02:167598] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n02:167598] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n02:167598] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n02:167598] [10] [b11n04:166205] *** Process received signal *** +[b11n04:166205] Signal: Segmentation fault (11) +[b11n04:166205] Signal code: (3) +[b11n04:166205] Failing at address: 0xb170000000000a0 +[b11n04:166205] [ 0] [0x3fffd5fe5150] +[b18n08:131380] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n08:131380] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n08:131380] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n08:131380] [ 5] [b21n08:116175] *** Process received signal *** +[b21n08:116175] Signal: Segmentation fault (11) +[b21n08:116175] Signal code: (3) +[b21n08:116175] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n06:127276] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n06:127276] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n06:127276] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n06:127276] [13] [b06n16:165925] *** Process received signal *** +[b06n16:165925] Signal: Segmentation fault (11) +[b06n16:165925] Signal code: (3) +[b06n16:165925] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n04:52707] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n04:52707] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n04:52707] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n07:117359] *** End of error message *** +[0x1000000504d8] +[b18n17:124163] [ 1] [0x3fffd9c6aaa0] +[b18n17:124163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n17:124163] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n17:124163] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n07:133373] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n07:133373] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n07:133373] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n07:133373] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n02:176050] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n02:176050] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n02:176050] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n02:176050] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n02:176050] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n12:132100] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n12:132100] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n12:132100] *** End of error message *** +[a02n04:41672] [ 0] [0x1000000504d8] +[a02n04:41672] [ 1] [0x3fffca3acd60] +[a02n04:41672] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n04:41672] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n04:41672] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n04:41672] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n16:127353] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n16:127353] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n16:127353] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n18:07421] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n18:07421] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n18:07421] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n11:23061] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n06:167367] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n06:167367] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n06:167367] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n06:167367] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n14:168375] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n14:168375] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n14:168375] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n14:168375] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n08:172900] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n08:172900] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n09:149100] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n09:149100] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n09:149100] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n09:149100] *** End of error message *** +[b22n13:122029] *** Process received signal *** +[b22n13:122029] Signal: Segmentation fault (11) +[b22n13:122029] Signal code: (3) +[b22n13:122029] Failing at address: 0xb170000000000a0 +[b07n11:170894] *** Process received signal *** +[b07n11:170894] Signal: Segmentation fault (11) +[b07n11:170894] Signal code: (3) +[b07n11:170894] Failing at address: 0xb170000000000a0 +[b07n11:170894] [ 0] [0x1000000504d8] +[b07n11:170894] [ 1] [0x3ffff56a3370] +[b07n11:170894] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n03:123614] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n03:123614] [10] [b20n01:128831] *** Process received signal *** +[b20n01:128831] Signal: Segmentation fault (11) +[b20n01:128831] Signal code: (3) +[b20n01:128831] Failing at address: 0x14170000000000a0 +[b20n01:128831] [ 0] [0x1000000504d8] +[b20n01:128831] [ 1] [0x3ffff5df4050] +[a02n03:38503] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n03:38503] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n03:38503] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n03:38503] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n03:38503] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n03:38503] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n03:38503] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n03:38503] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n07:180194] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n07:180194] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n07:180194] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n07:180194] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n07:180194] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n07:180194] *** End of error message *** +[b19n11:133968] *** Process received signal *** +[b19n11:133968] Signal: Segmentation fault (11) +[b19n11:133968] Signal code: (3) +[b19n11:133968] Failing at address: 0xb170000000000a0 +[b19n11:133968] [ 0] [0x1000000504d8] +[b19n11:133968] [ 1] [0x3fffe9fd3a40] +[b19n11:133968] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n11:133968] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n11:160348] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n11:160348] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n11:160348] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n11:160348] [13] [a27n01:48802] [ 0] [0x1000000504d8] +[a27n01:48802] [ 1] [0x3fffd9a94e20] +[a27n01:48802] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n01:48802] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n02:06563] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n02:06563] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n12:12765] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n12:12765] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n12:12765] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n12:12765] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n12:167678] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n12:167678] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n09:51334] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n09:51334] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n09:51334] [ 5] [b21n16:116523] *** Process received signal *** +[b21n16:116523] Signal: Segmentation fault (11) +[b21n16:116523] Signal code: (3) +[b21n16:116523] Failing at address: 0xb170000000000a0 +[b21n16:116523] [ 0] [a29n15:177055] *** Process received signal *** +[a29n15:177055] Signal: Segmentation fault (11) +[a29n15:177055] Signal code: (3) +[a29n15:177055] Failing at address: 0xb170000000000a0 +[a29n15:177055] [ 0] [a29n16:157885] [ 0] [0x1000000504d8] +[a29n16:157885] [ 1] [0x3ffff3b6e830] +[a29n16:157885] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n16:157885] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n16:157885] [ 4] [a36n10:142932] *** Process received signal *** +[a36n10:142932] Signal: Segmentation fault (11) +[a36n10:142932] Signal code: (3) +[a36n10:142932] Failing at address: 0x14170000000000a0 +[a36n10:142932] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n17:149907] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n17:149907] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n17:149907] [ 9] [0x3fffcacf5ca0] +[b19n06:126379] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n06:126379] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n09:123419] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n09:123419] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n09:123419] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n09:123419] [12] [0x3ffff2142870] +[a31n15:155430] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n15:155430] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n15:155430] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n15:155430] [ 5] [0x1000000504d8] +[b23n13:114395] [ 1] [0x3fffe3d29140] +[b23n13:114395] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n13:114395] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n13:114395] [ 4] [b23n07:118163] [ 0] [0x1000000504d8] +[b23n07:118163] [ 1] [0x3fffc3b09380] +[b23n07:118163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n07:118163] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n05:135165] *** End of error message *** +[b01n18:179696] *** Process received signal *** +[b01n18:179696] Signal: Segmentation fault (11) +[b01n18:179696] Signal code: (3) +[b01n18:179696] Failing at address: 0xb170000000000a0 +[b18n14:128868] *** Process received signal *** +[b18n14:128868] Signal: Segmentation fault (11) +[b18n14:128868] Signal code: (3) +[b18n14:128868] Failing at address: 0xb170000000000a0 +[b18n14:128868] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n14:133635] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n14:133635] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n12:122964] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n12:122964] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n12:122964] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n12:122964] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n12:122964] [14] [b14n13:122142] *** Process received signal *** +[b14n13:122142] Signal: Segmentation fault (11) +[b14n13:122142] Signal code: (3) +[b14n13:122142] Failing at address: 0xb170000000000a0 +[b14n13:122142] [ 0] [0x1000000504d8] +[b14n13:122142] [ 1] [0x3fffc1af6c00] +[b14n13:122142] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n13:122142] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n13:122142] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n13:122142] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n13:122142] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n13:122142] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n13:122142] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n13:122142] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n13:122142] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n13:122142] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n13:122142] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n13:122142] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n13:122142] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n13:122142] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n01:167297] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n01:167297] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n01:167297] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n14:169521] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n14:169521] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n17:15874] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n17:15874] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n17:15874] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n17:15874] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n13:176367] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n13:176367] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n13:176367] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n13:176367] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n01:141819] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n01:141819] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n01:141819] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n01:141819] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n09:03926] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n08:147822] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n08:147822] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n17:138141] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n17:138141] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n17:138141] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n17:138141] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n17:138141] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n17:138141] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n03:177925] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n03:177925] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n10:176309] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n10:176309] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n10:176309] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n10:176309] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n10:176309] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n10:176309] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n02:163029] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n02:163029] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n02:163029] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n02:163029] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n02:163029] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n02:163029] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n08:08955] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n08:08955] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n08:08955] [ 9] [a36n06:143008] *** Process received signal *** +[a36n06:143008] Signal: Segmentation fault (11) +[a36n06:143008] Signal code: (3) +[a36n06:143008] Failing at address: 0xb170000000000a0 +[a36n06:143008] [ 0] [0x1000000504d8] +[a36n06:143008] [ 1] [0x3ffffa0f5010] +[a36n06:143008] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n06:143008] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n14:159238] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n14:159238] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n14:159238] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n14:159238] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n06:43019] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n06:43019] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n06:43019] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n06:43019] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n06:43019] *** End of error message *** +[a35n14:136659] *** Process received signal *** +[a35n14:136659] Signal: Segmentation fault (11) +[a35n14:136659] Signal code: (3) +[a35n14:136659] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n04:169913] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n04:169913] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n04:169913] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n04:169913] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n04:169913] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n04:169913] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n04:169913] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n18:126904] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n18:126904] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n18:126904] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n18:126904] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n02:166747] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n02:166747] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n02:166747] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n02:166747] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n02:166747] [11] [b03n06:01839] *** Process received signal *** +[b03n06:01839] Signal: Segmentation fault (11) +[b03n06:01839] Signal code: (3) +[b03n06:01839] Failing at address: 0x14170000000000a0 +[b03n06:01839] [ 0] [0x1000000504d8] +[b03n06:01839] [ 1] [a36n07:153174] [ 0] [0x1000000504d8] +[a36n07:153174] [ 1] [0x3ffff583a870] +[a36n07:153174] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n07:153174] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n07:153174] [ 4] [b20n07:132086] [ 0] [0x1000000504d8] +[b20n07:132086] [ 1] [0x3ffffbbbe690] +[b20n07:132086] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n07:132086] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n02:164489] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n02:164489] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n02:164489] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n07:138201] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n07:138201] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n07:138201] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n07:131672] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n07:131672] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n07:131672] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n07:131672] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n07:131672] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n07:131672] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n07:131672] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n07:131672] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n11:171965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n11:171965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n11:171965] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n11:171965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n17:150928] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n17:150928] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n17:150928] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n17:150928] [10] [b02n16:01569] *** Process received signal *** +[b02n16:01569] Signal: Segmentation fault (11) +[b02n16:01569] Signal code: (3) +[b02n16:01569] Failing at address: 0x14170000000000a0 +[b18n01:128154] *** Process received signal *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n13:123708] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n13:123708] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n13:123708] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n13:123708] *** End of error message *** +[a28n03:10288] *** Process received signal *** +[a28n03:10288] Signal: Segmentation fault (11) +[a28n03:10288] Signal code: (3) +[a28n03:10288] Failing at address: 0xb170000000000a0 +[a28n03:10288] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n09:121755] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n09:121755] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n09:121755] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n09:121755] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n11:137873] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n11:137873] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n11:137873] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n11:137873] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n11:137873] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n11:152985] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n11:152841] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n11:152841] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n11:152841] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n11:152841] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n11:152841] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n18:28216] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n18:28216] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n18:28216] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n18:28216] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n11:01171] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n11:01171] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n08:124722] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n09:173749] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n09:173749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n09:173749] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n15:85942] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n15:85942] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n13:01298] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n13:01298] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n13:01298] *** End of error message *** +[0x3fffe8e26c40] +[b06n08:174409] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n08:174409] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n08:174409] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n08:174409] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n08:174409] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n08:174409] [ 7] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n01:130409] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n01:130409] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n01:130409] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n14:161617] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n10:130580] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n02:143966] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n02:143966] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n02:143966] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n02:143966] [10] [a30n16:153292] *** Process received signal *** +[a30n16:153292] Signal: Segmentation fault (11) +[a30n16:153292] Signal code: (3) +[a30n16:153292] Failing at address: 0x14170000000000a0 +[a30n16:153292] [ 0] [0x1000000504d8] +[a30n16:153292] [ 1] [0x3fffdf485730] +[a30n16:153292] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n16:153292] [ 3] [b21n06:122162] [ 0] [0x1000000504d8] +[b21n06:122162] [ 1] [0x3fffef80ade0] +[b21n06:122162] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n06:122162] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n06:122162] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n06:122162] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n16:122494] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n16:122494] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n16:122494] *** End of error message *** +[0x1000000504d8] +[b05n12:11145] [ 1] [0x3fffc68282f0] +[b05n12:11145] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n12:11145] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n12:11145] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n12:11145] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n12:11145] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n12:11145] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n12:11145] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n12:11145] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n12:11145] [10] [0x1000000504d8] +[b14n09:119546] [ 1] [0x3fffc67a51c0] +[b14n09:119546] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n09:119546] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n09:119546] [ 4] [0x3fffc69c0f20] +[b18n09:123811] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n09:123811] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n09:123811] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n09:123811] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n09:123811] [ 6] [0x1000000504d8] +[b08n16:171916] [ 1] [0x3fffe7bb4450] +[b08n16:171916] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n16:171916] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n16:171916] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n16:171916] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n12:148382] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n12:148382] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n12:148382] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n12:148382] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n12:123234] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n12:123234] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n12:123234] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n12:123234] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n01:173782] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n01:173782] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n01:173782] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n03:127629] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n03:127629] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n17:119583] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n17:119583] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n17:119583] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n17:119583] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n17:119583] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n12:156917] [13] [0x3fffe10ebdb0] +[b21n02:120188] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n02:120188] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n02:120188] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n01:145733] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n01:145733] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n01:145733] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n04:176000] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n15:148579] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n15:148579] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n15:148579] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n15:148579] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n04:138159] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n04:138159] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n04:138159] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n04:138159] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n02:163321] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n02:163321] *** End of error message *** +[b18n10:122205] *** Process received signal *** +[b18n10:122205] Signal: Segmentation fault (11) +[b18n10:122205] Signal code: (3) +[b18n10:122205] Failing at address: 0xb170000000000a0 +[b18n10:122205] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n17:164560] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n17:164560] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n17:164560] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n17:164560] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n17:164560] *** End of error message *** +[a36n18:163110] *** Process received signal *** +[a36n18:163110] Signal: Segmentation fault (11) +[a36n18:163110] Signal code: (3) +[a36n18:163110] Failing at address: 0x14170000000000a0 +[a36n18:163110] [ 0] [0x1000000504d8] +[a36n18:163110] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n09:122069] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n09:122069] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n09:122069] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n09:122069] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n09:122069] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n07:125473] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n07:125473] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n07:125473] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n07:125473] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n14:173895] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n14:173895] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n08:163161] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n08:163161] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n04:164047] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n04:164047] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n04:164047] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n03:168208] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n03:168208] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n03:168208] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n07:127354] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n07:127354] *** End of error message *** +[a34n11:147713] [ 0] [0x1000000504d8] +[a34n11:147713] [ 1] [0x3ffff17a43e0] +[a34n11:147713] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n11:147713] [ 3] [b06n01:165163] *** Process received signal *** +[b06n01:165163] Signal: Segmentation fault (11) +[b06n01:165163] Signal code: (3) +[b06n01:165163] Failing at address: 0xb170000000000a0 +[b06n01:165163] [ 0] [0x1000000504d8] +[b06n01:165163] [ 1] [0x3ffff8426400] +[b06n01:165163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n01:165163] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n01:165163] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n08:156371] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n08:156371] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n13:163194] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n13:163194] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n13:163194] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n13:163194] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n13:163194] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n13:165261] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n13:165261] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n13:165261] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n13:165261] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n11:176297] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n11:176297] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n11:176297] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n11:176297] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n11:176297] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n11:176297] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n11:176297] [10] [b23n12:115639] *** Process received signal *** +[a30n02:155382] *** Process received signal *** +[a30n02:155382] Signal: Segmentation fault (11) +[a30n02:155382] Signal code: (3) +[a30n02:155382] Failing at address: 0x14170000000000a0 +[a30n02:155382] [ 0] [0x1000000504d8] +[a30n02:155382] [ 1] [0x3fffeac98100] +[a30n02:155382] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n02:155382] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n08:163099] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n08:163099] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n08:163099] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n08:163099] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n11:152817] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n11:152817] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n11:152817] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n12:177119] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n12:177119] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n12:177119] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n12:177119] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n12:177119] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n05:169785] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n15:154383] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n15:154383] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n05:148377] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n05:148377] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n05:148377] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n08:121690] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n08:121690] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n08:121690] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n15:137008] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n15:137008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n15:137008] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n15:137008] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n15:137008] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n03:39683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n03:39683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n03:39683] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n03:39683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n03:39683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n03:39683] [ 9] [b03n11:175597] *** Process received signal *** +[b03n11:175597] Signal: Segmentation fault (11) +[b03n11:175597] Signal code: (3) +[b03n11:175597] Failing at address: 0x14170000000000a0 +[b03n11:175597] [ 0] [0x1000000504d8] +[b03n11:175597] [ 1] [0x3fffd6a3b740] +[b03n11:175597] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n11:175597] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n02:149227] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n02:149227] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n09:79006] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n09:79006] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n09:79006] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n11:120241] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n04:169409] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n04:169409] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n01:138003] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n01:138003] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n01:138003] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n01:138003] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n17:146794] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n17:146794] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n17:146794] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n17:146794] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n06:10224] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n06:10224] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n06:10224] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n06:10224] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n06:10224] *** End of error message *** +[b02n11:02683] *** Process received signal *** +[b02n11:02683] Signal: Segmentation fault (11) +[b02n11:02683] Signal code: (3) +[b02n11:02683] Failing at address: 0x14170000000000a0 +[b02n11:02683] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n12:140121] [ 6] [b15n17:126524] *** Process received signal *** +[b15n17:126524] Signal: Segmentation fault (11) +[b15n17:126524] Signal code: (3) +[b15n17:126524] Failing at address: 0x5170000000000a0 +[b11n12:154345] *** Process received signal *** +[b11n12:154345] Signal: Segmentation fault (11) +[b11n12:154345] Signal code: (3) +[b11n12:154345] Failing at address: 0xb170000000000a0 +[b11n12:154345] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n09:164364] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n03:168932] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n03:168932] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n03:168932] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n03:168932] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n03:168932] [ 9] [b14n15:123037] *** Process received signal *** +[b14n15:123037] Signal: Segmentation fault (11) +[b14n15:123037] Signal code: (3) +[b14n15:123037] Failing at address: 0x14170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n14:163550] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n14:163550] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n14:163550] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n14:163550] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n12:11419] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n12:11419] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n12:11419] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n12:11419] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n12:11419] [a35n03:135162] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n15:161757] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n15:161757] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n15:161757] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n15:161757] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n01:153682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n01:153682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n01:153682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n01:153682] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n17:119467] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n14:21316] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n14:21316] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n14:21316] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n14:21316] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n14:21316] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n16:124793] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n16:124793] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n16:124793] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n03:167779] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n03:167779] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n03:167779] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n03:167779] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n03:167779] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n03:167779] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n03:167779] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n03:167779] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n03:167779] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n03:167779] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n03:167779] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n03:167779] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n06:177004] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n06:177004] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n06:177004] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n01:65728] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n01:65728] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n01:65728] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n01:65728] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n01:65728] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n01:65728] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n01:65728] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n01:65728] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n01:65728] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n01:65728] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n03:136668] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n03:136668] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n06:154623] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n06:154623] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n06:154623] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n05:162566] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n05:162566] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n03:125241] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n03:125241] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n03:125241] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n03:125241] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n04:169704] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n04:169704] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n04:169704] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n04:169704] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n04:169704] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n04:169704] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n09:147790] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n11:148675] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n11:148675] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n11:148675] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n11:148675] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n03:01354] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n03:01354] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n04:118345] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n02:178474] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n02:178474] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n02:178474] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n02:178474] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n02:178474] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n02:178474] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n02:122556] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n02:122556] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n02:122556] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n02:122556] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n02:122556] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n02:127219] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n02:127219] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n02:127219] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n02:127219] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n02:127219] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n02:127219] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n10:161716] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n10:161716] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n10:161716] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n09:163519] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n09:163519] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n09:163519] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n09:163519] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n09:163519] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n09:163519] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n02:123704] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n02:123704] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n02:123704] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n02:123704] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n02:123704] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n17:162358] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n17:162358] [ 6] [0x3fffd92a3120] +[a34n01:164964] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n01:164964] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n01:164964] [ 4] [b20n06:130541] *** Process received signal *** +[b20n06:130541] Signal: Segmentation fault (11) +[b20n06:130541] Signal code: (3) +[b20n06:130541] Failing at address: 0xb170000000000a0 +[b20n06:130541] [ 0] [0x1000000504d8] +[b20n06:130541] [ 1] [0x3fffdc3e9750] +[b20n06:130541] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n06:130541] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n17:163816] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n05:131261] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n05:131261] *** End of error message *** +[a28n04:13080] *** Process received signal *** +[a28n04:13080] Signal: Segmentation fault (11) +[a28n04:13080] Signal code: (3) +[a28n04:13080] Failing at address: 0x14170000000000a0 +[a28n04:13080] [ 0] [0x1000000504d8] +[a28n04:13080] [ 1] [0x3ffff1c6d380] +[a28n04:13080] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n04:13080] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n06:133953] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n06:133953] *** End of error message *** +[b18n05:135119] *** Process received signal *** +[b18n05:135119] Signal: Segmentation fault (11) +[b18n05:135119] Signal code: (3) +[b18n05:135119] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n03:151880] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n03:151880] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n03:151880] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n03:151880] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n01:179263] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n01:179263] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n01:179263] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n02:147885] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n02:147885] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n02:147885] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n02:147885] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n08:172426] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n08:172426] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n08:172426] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n08:175804] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n08:175804] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n08:175804] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n08:175804] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n08:175804] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n08:175804] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n08:175804] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n08:175804] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n08:175804] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n08:175804] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n05:54535] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n05:54535] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n05:54535] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n01:171785] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n01:171785] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n17:118575] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n17:118575] [10] [0x1000000504d8] +[a32n05:154189] [ 1] [0x3fffd400d1c0] +[a32n05:154189] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n05:154189] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n05:154189] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n04:142254] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n04:142254] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n04:142254] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n04:142254] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n05:130803] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n05:130803] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n05:130803] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n05:130803] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n05:130803] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n05:130803] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n05:130803] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n03:153850] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n03:153850] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n03:153850] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n01:141341] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n01:141341] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n01:141341] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n01:141341] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n18:118681] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n18:118681] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n18:118681] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n11:164994] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n11:164994] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n16:121929] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n16:121929] [ 5] [a29n10:164937] *** Process received signal *** +[a29n10:164937] Signal: Segmentation fault (11) +[a29n10:164937] Signal code: (3) +[a29n10:164937] Failing at address: 0xb170000000000a0 +[a29n10:164937] [ 0] [b10n06:162916] *** Process received signal *** +[b10n06:162916] Signal: Segmentation fault (11) +[b10n06:162916] Signal code: (3) +[b10n06:162916] Failing at address: 0xb170000000000a0 +[b10n06:162916] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n08:164752] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n08:164752] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n08:164752] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n08:164752] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n08:164752] [14] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n14:152036] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n01:132062] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n01:132062] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n01:132062] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n01:132062] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n01:132062] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n04:04850] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n04:04850] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n04:04850] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n04:04850] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n04:04850] *** End of error message *** +[b14n03:130349] *** Process received signal *** +[b14n03:130349] Signal: Segmentation fault (11) +[b14n03:130349] Signal code: (3) +[b14n03:130349] Failing at address: 0xb170000000000a0 +[b14n03:130349] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n06:132106] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n06:132106] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n06:132106] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n06:132106] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n06:132106] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n06:171504] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n06:171504] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n06:171504] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n06:171504] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n06:171504] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n06:120733] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n06:120733] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n06:120733] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n06:120733] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n06:120733] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n06:120733] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n06:120733] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n06:120733] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n06:120733] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n06:120733] *** End of error message *** +[b23n02:116888] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n02:116888] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n02:116888] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n02:116888] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n02:116888] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n02:116888] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n02:116888] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n02:116888] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n02:116888] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n07:03372] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n07:03372] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n07:03372] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n07:03372] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n05:173080] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n02:139023] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n02:139023] *** End of error message *** +[b06n06:163675] *** Process received signal *** +[b06n06:163675] Signal: Segmentation fault (11) +[b06n06:163675] Signal code: (3) +[b06n06:163675] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n11:160798] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n11:160798] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n11:160798] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n11:160798] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n11:160798] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n01:128004] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n01:128004] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n01:128004] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n05:120332] *** End of error message *** +[b12n12:138735] *** Process received signal *** +[b12n12:138735] Signal: Segmentation fault (11) +[b12n12:138735] Signal code: (3) +[b12n12:138735] Failing at address: 0xb170000000000a0 +[b12n12:138735] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n16:164550] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n16:164550] [ 5] [0x1000000504d8] +[b14n04:126857] [ 1] [0x3ffffc21c740] +[b14n04:126857] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n04:126857] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n04:126857] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n17:123234] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n17:123234] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n17:123234] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n17:123234] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n17:123234] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n17:123234] [ 9] [b14n05:126779] *** Process received signal *** +[b14n05:126779] Signal: Segmentation fault (11) +[b14n05:126779] Signal code: (3) +[b14n05:126779] Failing at address: 0xb170000000000a0 +[b14n05:126779] [ 0] [0x3fffed8945e0] +[b05n13:10196] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n13:10196] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n13:10196] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n13:10196] [ 5] [b15n07:124348] *** Process received signal *** +[b15n07:124348] Signal: Segmentation fault (11) +[b15n07:124348] Signal code: (3) +[b15n07:124348] Failing at address: 0x14170000000000a0 +[b15n07:124348] [ 0] [b10n01:165122] *** Process received signal *** +[b10n01:165122] Signal: Segmentation fault (11) +[b10n01:165122] Signal code: (3) +[b10n01:165122] Failing at address: 0x14170000000000a0 +[b10n01:165122] [ 0] [0x1000000504d8] +[a32n09:155760] [ 1] [0x3fffe7d81c10] +[a32n09:155760] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n09:155760] [ 3] [a30n17:176412] *** Process received signal *** +[a30n17:176412] Signal: Segmentation fault (11) +[a30n17:176412] Signal code: (3) +[a30n17:176412] Failing at address: 0xb170000000000a0 +[a30n17:176412] [ 0] [0x1000000504d8] +[a30n17:176412] [ 1] [0x3fffcfb7e610] +[a30n17:176412] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n17:176412] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n06:157789] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n06:157789] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n06:157789] [14] [0x3fffde69d040] +[b11n13:153676] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n13:153676] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n13:153676] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n13:153676] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n13:153676] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n13:153676] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n13:153676] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n13:153676] [ 9] [0x1000000504d8] +[b23n11:117472] [ 1] [0x3ffff8f99a90] +[b23n11:117472] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n11:117472] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n11:117472] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n12:178599] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n12:178599] [ 6] [0x3fffea1448f0] +[b08n06:161723] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n06:161723] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n06:161723] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n06:161723] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n14:178325] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n14:178325] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n14:178325] [ 6] [0x3fffe7acd110] +[a33n17:168154] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n17:168154] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n17:168154] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n17:168154] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n07:152083] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n07:152083] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n06:117526] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n06:117526] *** End of error message *** +[0x3fffe5b0bbf0] +[b07n10:16973] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n10:16973] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n10:16973] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n10:16973] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n10:16973] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n11:03985] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n11:03985] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n11:03985] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n11:03985] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n11:03985] [14] [0x3fffed1616f0] +[a29n07:160800] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n07:160800] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n07:160800] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n03:120667] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n03:120667] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n03:46690] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n03:46690] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n03:46690] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n03:46690] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n03:46690] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n03:46690] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n03:46690] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n03:46690] *** End of error message *** +[b21n11:120584] *** Process received signal *** +[b21n11:120584] Signal: Segmentation fault (11) +[b21n11:120584] Signal code: (3) +[b21n11:120584] Failing at address: 0x14170000000000a0 +[b21n11:120584] [ 0] [a31n04:158631] *** Process received signal *** +[a31n04:158631] Signal: Segmentation fault (11) +[a31n04:158631] Signal code: (3) +[a31n04:158631] Failing at address: 0x14170000000000a0 +[a31n04:158631] [ 0] [0x1000000504d8] +[a31n04:158631] [ 1] [0x3fffe2ac6730] +[a31n04:158631] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n04:158631] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n03:154294] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n03:154294] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n06:138763] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n06:138763] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n06:138763] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n06:138763] [ 7] [0x3ffff323ba20] +[a35n04:145748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n04:145748] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n04:145748] [ 4] [b23n09:119632] *** Process received signal *** +[b23n09:119632] Signal: Segmentation fault (11) +[b23n09:119632] Signal code: (3) +[b23n09:119632] Failing at address: 0xb170000000000a0 +[b23n09:119632] [ 0] [b02n13:171106] *** Process received signal *** +[b02n13:171106] Signal: Segmentation fault (11) +[b02n13:171106] Signal code: (3) +[b02n13:171106] Failing at address: 0xb170000000000a0 +[b02n13:171106] [ 0] [a33n05:139437] *** Process received signal *** +[a33n05:139437] Signal: Segmentation fault (11) +[a33n05:139437] Signal code: (3) +[a33n05:139437] Failing at address: 0x14170000000000a0 +[a33n05:139437] [ 0] [0x1000000504d8] +[a33n05:139437] [ 1] [0x3fffe75f8fd0] +[a33n05:139437] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n18:179170] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n18:179170] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n18:179170] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n18:179170] [ 7] [b17n13:123514] [ 0] [0x1000000504d8] +[b17n13:123514] [ 1] [0x3fffca2eb780] +[b17n13:123514] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n13:123514] [ 3] [b08n07:168961] *** Process received signal *** +[b08n07:168961] Signal: Segmentation fault (11) +[b08n07:168961] Signal code: (3) +[b08n07:168961] Failing at address: 0x14170000000000a0 +[b08n07:168961] [ 0] [0x1000000504d8] +[b08n07:168961] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n13:154230] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n13:154230] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n13:154230] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n14:08399] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n14:08399] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n04:44756] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n17:169735] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n17:169735] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n17:169735] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n17:169735] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n17:169735] [ 9] [b14n12:123561] [ 0] [0x1000000504d8] +[b14n12:123561] [ 1] [0x3fffc0f63270] +[b14n12:123561] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n12:123561] [ 3] [b03n03:06823] [ 0] [0x1000000504d8] +[b03n03:06823] [ 1] [0x3fffd2276ef0] +[b03n03:06823] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n03:06823] [ 3] [0x1000000504d8] +[a36n05:139913] [ 1] [0x3fffe0038df0] +[a36n05:139913] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n05:139913] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n05:139913] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n05:139913] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n05:156008] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n05:156008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n05:156008] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n05:156008] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n05:156008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n05:156008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n05:156008] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n05:156008] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n05:156008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n05:156008] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n14:133970] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n14:133970] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n14:133970] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n14:133970] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n14:133970] [ 9] [0x3fffc817da40] +[a30n10:169210] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n10:169210] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n10:169210] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n10:169210] [ 5] [0x1000000504d8] +[a29n05:174780] [ 1] [0x3fffdbbbbe30] +[a29n05:174780] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n05:174780] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n05:174780] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n05:174780] [ 5] [b18n13:123051] Signal: Segmentation fault (11) +[b18n13:123051] Signal code: (3) +[b18n13:123051] Failing at address: 0xb170000000000a0 +[b18n13:123051] [ 0] [0x1000000504d8] +[b18n13:123051] [ 1] [0x3fffca96f2b0] +[b18n13:123051] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n13:123051] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n11:134518] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n11:134518] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n11:134518] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n11:134518] [ 8] [0x1000000504d8] +[b08n10:11491] [ 1] [0x3ffff22f42c0] +[b08n10:11491] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n10:11491] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n10:11491] [ 4] [ 1] [0x3fffc4ac5fa0] +[a30n01:163390] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n01:163390] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n10:118390] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n10:118390] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n10:118390] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n08:141874] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n08:141874] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n08:141874] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n08:141874] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n04:03147] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n04:03147] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n04:03147] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n04:03147] [ 9] [0x3fffe7d064a0] +[b07n12:160962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n12:160962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n12:160962] [ 4] [b09n08:176683] *** Process received signal *** +[b09n08:176683] Signal: Segmentation fault (11) +[b09n08:176683] Signal code: (3) +[b09n08:176683] Failing at address: 0x14170000000000a0 +[b09n08:176683] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n12:145496] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n12:145496] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n12:145496] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n12:145496] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n12:145496] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n12:145496] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n01:96754] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n01:96754] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n01:96754] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n01:96754] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n01:96754] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n09:12245] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n09:12245] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n09:12245] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n09:12245] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n09:12245] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n11:130804] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n11:130804] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n11:130804] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n11:130804] [ 8] [b23n05:120228] *** Process received signal *** +[b23n05:120228] Signal: Segmentation fault (11) +[b23n05:120228] Signal code: (3) +[b23n05:120228] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n01:170767] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n01:170767] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n01:170767] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n10:140415] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n10:140415] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n10:140415] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n10:140415] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n10:140415] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n16:154183] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n16:154183] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n16:154183] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n06:179454] [ 4] [0x3fffd2be0f50] +[b07n03:172650] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n03:172650] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n03:172650] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n03:172650] [ 5] [0x1000000504d8] +[b13n18:128544] [ 1] [0x3fffe49b8ca0] +[b13n18:128544] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n18:128544] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n18:128544] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n18:128544] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n18:128544] [ 6] [a29n08:159025] *** Process received signal *** +[a29n08:159025] Signal: Segmentation fault (11) +[a29n08:159025] Signal code: (3) +[a29n08:159025] Failing at address: 0x14170000000000a0 +[a29n08:159025] [ 0] [0x1000000504d8] +[a29n08:159025] [ 1] [0x3fffdbe218b0] +[a29n08:159025] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n08:159025] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n08:159025] [ 4] [b18n15:120129] *** Process received signal *** +[b18n15:120129] Signal: Segmentation fault (11) +[b18n15:120129] Signal code: (3) +[b18n15:120129] Failing at address: 0xb170000000000a0 +[b13n13:133627] *** Process received signal *** +[b13n13:133627] Signal: Segmentation fault (11) +[b13n13:133627] Signal code: (3) +[b13n13:133627] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n14:152747] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n10:126612] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n10:126612] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n10:126612] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n10:126612] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n10:126612] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n10:126612] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n10:126612] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n10:126612] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n10:126612] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n10:126612] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n01:124278] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n01:124278] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n11:156154] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n11:156154] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n08:176175] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n08:176175] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n08:176175] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n08:176175] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n08:176175] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n06:119115] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n06:119115] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n02:122181] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n02:122181] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n02:122181] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n02:122181] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n15:139747] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n15:139747] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n15:139747] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n15:139747] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n15:139747] *** End of error message *** +[a28n05:04517] *** Process received signal *** +[a28n05:04517] Signal: Segmentation fault (11) +[a28n05:04517] Signal code: (3) +[a28n05:04517] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n15:122084] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n15:122084] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n15:122084] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n15:122084] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n15:122084] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n07:165091] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n07:165091] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n07:165091] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n07:165091] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n07:165091] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n07:165091] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n07:165091] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n07:165091] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n07:165091] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n08:127446] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n08:127446] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n18:10724] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n09:168244] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n09:168244] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n09:160373] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n09:160373] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n09:160373] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n09:160373] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n09:160373] [14] [b05n10:10926] [ 0] [0x1000000504d8] +[b05n10:10926] [ 1] [0x3fffc45debf0] +[b05n10:10926] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n10:10926] [ 3] [0x1000000504d8] +[a31n10:155400] [ 1] [0x3fffe608cd00] +[a31n10:155400] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n10:155400] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n10:155400] [ 4] [0x3ffff4849f60] +[a02n10:135850] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n10:135850] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n10:135850] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n10:135850] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n10:135850] [ 6] [b19n01:121745] *** Process received signal *** +[b19n01:121745] Signal: Segmentation fault (11) +[b19n01:121745] Signal code: (3) +[b19n01:121745] Failing at address: 0x14170000000000a0 +[b19n01:121745] [ 0] [b09n10:169680] *** Process received signal *** +[b09n10:169680] Signal: Segmentation fault (11) +[b09n10:169680] Signal code: (3) +[b09n10:169680] Failing at address: 0x14170000000000a0 +[b09n10:169680] [ 0] [a26n02:74430] *** Process received signal *** +[a26n02:74430] Signal: Segmentation fault (11) +[a26n02:74430] Signal code: (3) +[a26n02:74430] Failing at address: 0x14170000000000a0 +[a26n02:74430] [ 0] [0x1000000504d8] +[a26n02:74430] [ 1] [0x3ffff7307a20] +[a26n02:74430] [ 2] [ 0] [0x1000000504d8] +[a32n07:172879] [ 1] [0x3fffe7880c40] +[a32n07:172879] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n07:172879] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n10:120978] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n10:120978] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n10:120978] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n03:164494] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n03:164494] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n03:164494] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n03:164494] [ 9] [0x3fffdb5fe970] +[b19n04:130747] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n04:130747] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n04:130747] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n04:130747] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n04:130747] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n04:130747] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n04:130747] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n04:130747] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n04:130747] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n04:130747] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n04:130747] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n04:130747] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n04:130747] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n04:130747] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n16:124961] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n16:124961] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n16:124961] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n16:124961] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n13:155850] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n13:155850] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n05:149465] *** End of error message *** +[a27n07:45309] *** Process received signal *** +[a27n07:45309] Signal: Segmentation fault (11) +[a27n07:45309] Signal code: (3) +[a27n07:45309] Failing at address: 0x14170000000000a0 +[a27n07:45309] [ 0] [0x1000000504d8] +[a27n07:45309] [ 1] [0x3fffd8b15610] +[a27n07:45309] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n07:45309] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n12:122875] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n12:122875] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n12:122875] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n12:160190] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n12:160190] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n09:106787] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n09:106787] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n09:106787] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n09:106787] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n09:106787] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n09:106787] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n09:106787] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n09:106787] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n14:177512] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n14:177512] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n14:177512] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n14:177512] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n14:177512] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n14:177512] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n14:177512] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n14:177512] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n14:177512] [12] [0x1000000504d8] +[b18n02:125373] [ 1] [0x3fffe0c76820] +[b18n02:125373] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n02:125373] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n02:125373] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n09:170656] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n09:170656] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n05:176910] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n10:122757] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n10:122757] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n10:122757] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n10:122757] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n08:146116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n08:146116] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n08:146116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n08:146116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n04:156414] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n04:156414] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n04:156414] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n04:156414] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n04:156414] [13] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n07:156489] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n03:135038] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n03:135038] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n03:135038] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n09:128492] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n09:128492] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n09:128492] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n09:128492] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n09:128492] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n07:02840] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n07:02840] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n07:02840] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n17:163177] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n17:163177] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n10:176263] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n10:176263] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n10:176263] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n10:176263] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n10:176263] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n14:126118] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n14:126118] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n14:126118] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n14:126118] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n04:136997] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n03:126186] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n03:126186] *** End of error message *** +[0x1000000504d8] +[b15n08:118550] [ 1] [0x3fffea166720] +[b15n08:118550] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n08:118550] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n08:118550] [ 4] [a33n03:147533] *** Process received signal *** +[a33n03:147533] Signal: Segmentation fault (11) +[a33n03:147533] Signal code: (3) +[a33n03:147533] Failing at address: 0x5170000000000a0 +[a33n03:147533] [ 0] [0x1000000504d8] +[a33n03:147533] [ 1] [0x3fffeff0d350] +[a33n03:147533] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n03:147533] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n11:134748] *** End of error message *** +[a26n03:68829] *** Process received signal *** +[a26n03:68829] Signal: Segmentation fault (11) +[a26n03:68829] Signal code: (3) +[a26n03:68829] Failing at address: 0xb170000000000a0 +[a26n03:68829] [ 0] [b14n14:122441] *** Process received signal *** +[b14n14:122441] Signal: Segmentation fault (11) +[b14n14:122441] Signal code: (3) +[b14n14:122441] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n05:173823] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n01:123569] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n01:123569] [ 5] [a35n12:145266] *** Process received signal *** +[a35n12:145266] Signal: Segmentation fault (11) +[a35n12:145266] Signal code: (3) +[a35n12:145266] Failing at address: 0x5170000000000a0 +[a35n12:145266] [ 0] [0x1000000504d8] +[a35n12:145266] [ 1] [0x3fffc104f460] +[a35n12:145266] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n12:145266] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n08:165282] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n08:165282] *** End of error message *** +[0x1000000504d8] +[a34n13:137747] [ 1] [0x3fffc0ba8de0] +[a34n13:137747] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n13:137747] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n13:137747] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n13:137747] [ 5] [0x1000000504d8] +[b15n03:119830] [ 1] [0x3ffff07b6370] +[b15n03:119830] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n03:119830] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n03:119830] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n03:119830] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n13:165388] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n13:165388] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n13:165388] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n13:165388] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n13:165388] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n07:125450] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n07:125450] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n07:125450] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n07:125450] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n06:148311] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n06:148311] [ 6] [0x3ffffe985580] +[b17n08:124115] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n08:124115] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n08:124115] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n08:124115] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n04:136608] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n04:136608] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n04:136608] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n04:136608] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n04:136608] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n11:128954] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n11:128954] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n11:128954] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n11:128954] [14] [0x1000000504d8] +[b07n07:163252] [ 1] [0x3ffff62611e0] +[b07n07:163252] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n07:163252] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n07:163252] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n03:124682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n03:124682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n03:124682] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n12:120405] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n16:129236] [ 9] [0x1000000504d8] +[b03n14:177221] [ 1] [0x3fffde9bf890] +[b03n14:177221] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n14:177221] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n14:177221] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n14:177221] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n14:177221] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n07:156139] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n07:156139] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n07:156139] [12] [b18n18:121307] Signal: Segmentation fault (11) +[b18n18:121307] Signal code: (3) +[b18n18:121307] Failing at address: 0xb170000000000a0 +[b18n18:121307] [ 0] [0x1000000504d8] +[b18n18:121307] [ 1] [0x3fffc449cbd0] +[b18n18:121307] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n18:121307] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n14:116411] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n14:116411] [14] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n05:05605] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n05:05605] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n05:05605] [ 5] [a33n12:168642] *** Process received signal *** +[a33n12:168642] Signal: Segmentation fault (11) +[a33n12:168642] Signal code: (3) +[a33n12:168642] Failing at address: 0x14170000000000a0 +[a33n12:168642] [ 0] [0x1000000504d8] +[a33n12:168642] [ 1] [0x3fffe3d7e620] +[a33n12:168642] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n12:168642] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n12:168642] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n12:168642] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n12:168642] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n12:168642] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n12:168642] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n12:168642] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n12:168642] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n12:168642] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n12:168642] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n12:168642] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n12:168642] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n12:168642] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n04:177597] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n04:177597] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n04:177597] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n04:177597] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n04:177597] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n13:177638] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n13:177638] *** End of error message *** +[0x3fffea0184b0] +[b13n09:143897] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n09:143897] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n09:143897] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n09:143897] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n09:143897] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n09:143897] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n09:143897] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n14:139842] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n14:139842] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n04:117352] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n08:144854] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n08:144854] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n08:144854] [10] [b17n17:125805] *** Process received signal *** +[b17n17:125805] Signal: Segmentation fault (11) +[b17n17:125805] Signal code: (3) +[b17n17:125805] Failing at address: 0x14170000000000a0 +[ 1] [0x3ffffb993510] +[a28n12:177493] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n12:177493] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n12:177493] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n12:177493] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n12:177493] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n12:177493] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n12:177493] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n12:177493] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n08:138175] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n08:138175] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n05:53372] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n05:53372] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n13:54241] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n13:54241] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n12:180113] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n12:180113] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n12:180113] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n12:180113] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n12:180113] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n12:180113] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n18:122553] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n18:122553] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n18:122553] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n07:149475] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n07:149475] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n07:149475] [ 8] [0x1000000504d8] +[b20n05:121821] [ 1] [0x3fffe3cd9660] +[b20n05:121821] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n05:121821] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n05:121821] [ 4] [0x3fffd4290050] +[b19n16:130376] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n16:130376] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n16:130376] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n16:130376] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n10:126193] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n16:147286] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n16:147286] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n16:147286] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n16:147286] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n16:147286] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n04:168489] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n18:03311] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n18:03311] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n18:03311] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n18:03311] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n09:166573] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n09:166573] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n09:166573] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n09:166573] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n09:166573] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n15:148166] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n15:148166] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n13:138091] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n13:138091] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n13:138091] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n13:138091] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n13:138091] *** End of error message *** +[a27n15:32398] *** Process received signal *** +[a27n15:32398] Signal: Segmentation fault (11) +[a27n15:32398] Signal code: (3) +[a27n15:32398] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n10:124919] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n10:124919] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n10:124919] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n10:124919] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n10:124919] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n12:164383] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n12:164383] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n12:164383] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n12:164383] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n12:164383] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n14:136135] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n14:136135] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n14:136135] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n14:136135] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n14:136135] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n11:124446] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n11:124446] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n15:119255] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n15:119255] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n15:119255] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n15:119255] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n01:163023] [ 5] [0x3fffe68497e0] +[b22n15:120061] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n15:120061] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n15:120061] [ 4] [a36n15:151249] Signal: Segmentation fault (11) +[a36n15:151249] Signal code: (3) +[a36n15:151249] Failing at address: 0x5170000000000a0 +[a34n14:150130] *** Process received signal *** +[a34n14:150130] Signal: Segmentation fault (11) +[a34n14:150130] Signal code: (3) +[a34n14:150130] Failing at address: 0x14170000000000a0 +[a34n14:150130] [ 0] [0x1000000504d8] +[a34n14:150130] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n18:176808] *** End of error message *** +[b21n14:119486] *** Process received signal *** +[b21n14:119486] Signal: Segmentation fault (11) +[b21n14:119486] Signal code: (3) +[b21n14:119486] Failing at address: 0x14170000000000a0 +[b21n14:119486] [ 0] [0x1000000504d8] +[b21n14:119486] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n11:163261] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n11:163261] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n12:178717] [ 7] [b21n13:117507] *** Process received signal *** +[b21n13:117507] Signal: Segmentation fault (11) +[b21n13:117507] Signal code: (3) +[b21n13:117507] Failing at address: 0x14170000000000a0 +[b21n13:117507] [ 0] [0x1000000504d8] +[b21n13:117507] [ 1] [0x3fffe707d4c0] +[b21n13:117507] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n13:117507] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n04:153963] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n04:153963] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n04:153963] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n04:153963] [10] [0x3fffd5ec19b0] +[b13n10:134734] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n10:134734] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n10:134734] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n10:134734] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n17:168159] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n17:168159] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n17:168159] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n04:126344] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n04:126344] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n04:126344] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n04:126344] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n04:126344] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n04:126344] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n13:166924] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n13:166924] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n06:164014] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n06:164014] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n11:162469] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n11:162469] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n11:162469] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n02:119533] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n02:119533] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n02:119533] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n02:119533] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n02:119533] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n11:124432] [ 5] [a33n13:168158] *** Process received signal *** +[a33n13:168158] Signal: Segmentation fault (11) +[a33n13:168158] Signal code: (3) +[a33n13:168158] Failing at address: 0xb170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n03:158656] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n03:158656] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n03:158656] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n11:153416] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n11:153416] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n11:153416] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n11:153416] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n16:173550] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n16:173550] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n16:149062] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n16:149062] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n16:149062] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n16:149062] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n16:149062] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n05:125613] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n05:125613] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n05:125613] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n05:125613] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n05:125613] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n05:125613] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n18:164796] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n09:161231] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n18:08428] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n18:08428] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n10:173683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n10:173683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n10:173683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n10:173683] *** End of error message *** +[b13n17:129929] *** Process received signal *** +[b13n17:129929] Signal: Segmentation fault (11) +[b13n17:129929] Signal code: (3) +[b13n17:129929] Failing at address: 0x14170000000000a0 +[b19n15:124802] *** Process received signal *** +[b19n15:124802] Signal: Segmentation fault (11) +[b19n15:124802] Signal code: (3) +[b19n15:124802] Failing at address: 0x14170000000000a0 +[b19n15:124802] [ 0] [0x1000000504d8] +[b19n15:124802] [ 1] [0x3ffffff19f90] +[b19n15:124802] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n15:124802] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n15:124802] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n18:140387] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n18:140387] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n18:140387] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n18:140387] [ 6] [0x1000000504d8] +[b21n12:120128] [ 1] [0x3fffe3ea6aa0] +[b21n12:120128] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n12:120128] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n13:123885] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n13:123885] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n16:128438] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n16:128438] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n16:128438] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n18:131584] [ 9] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n05:164945] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n05:164945] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n05:164945] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n05:164945] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n08:54627] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n08:54627] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n08:54627] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n08:54627] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n08:54627] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n08:54627] [14] [a32n10:162607] *** Process received signal *** +[a32n10:162607] Signal: Segmentation fault (11) +[a32n10:162607] Signal code: (3) +[a32n10:162607] Failing at address: 0xb170000000000a0 +[b19n14:133064] *** Process received signal *** +[b19n14:133064] Signal: Segmentation fault (11) +[b19n14:133064] Signal code: (3) +[b19n14:133064] Failing at address: 0x14170000000000a0 +[b19n14:133064] [ 0] [0x1000000504d8] +[b19n14:133064] [ 1] [0x3fffe4363c40] +[b19n14:133064] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n14:133064] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n17:171198] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n17:171198] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n17:171198] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n02:167599] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n02:167599] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n02:167599] [12] [0x1000000504d8] +[b11n04:166204] [ 1] [0x3ffff8691d80] +[b11n04:166204] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n04:166204] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n04:166204] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n08:131379] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n08:131379] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n08:131379] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n08:131379] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n08:116174] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n08:116174] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n06:127275] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n06:127275] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n06:127275] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n06:127275] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n04:52707] [14] [b06n16:165925] [ 0] [0x1000000504d8] +[b06n16:165925] [ 1] [0x3fffe950c220] +[b06n16:165925] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n16:165925] [ 3] [b18n17:124164] *** Process received signal *** +[b18n17:124164] Signal: Segmentation fault (11) +[b18n17:124164] Signal code: (3) +[b18n17:124164] Failing at address: 0xb170000000000a0 +[b18n17:124164] [ 0] [0x1000000504d8] +[b18n17:124164] [ 1] [b21n07:117358] *** Process received signal *** +[b21n07:117358] Signal: Segmentation fault (11) +[b21n07:117358] Signal code: (3) +[b21n07:117358] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n07:133372] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n02:176051] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n02:176051] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n02:176051] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n02:176051] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n02:176051] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n12:132099] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n16:127352] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n04:41671] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n04:41671] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n04:41671] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n04:41671] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n04:41671] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n04:41671] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n18:07422] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n18:07422] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n18:07422] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n06:167368] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n11:23061] [ 7] [0x3fffc262ca00] +[b22n13:122030] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n13:122030] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n13:122030] [ 4] [b07n14:176069] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n14:168374] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n14:168374] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n03:123615] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n03:123615] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n03:123615] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n03:123615] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n03:38502] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n03:38502] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n03:38502] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n03:38502] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n03:38502] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n03:38502] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n03:38502] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n03:38502] [14] [b07n11:170895] *** Process received signal *** +[b07n11:170895] Signal: Segmentation fault (11) +[b07n11:170895] Signal code: (3) +[b07n11:170895] Failing at address: 0xb170000000000a0 +[b07n11:170895] [ 0] [0x1000000504d8] +[b07n11:170895] [ 1] [0x3fffc1e50790] +[b07n11:170895] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n11:170895] [ 3] [0x1000000504d8] +[b20n01:128830] [ 1] [0x3fffdcccb680] +[b20n01:128830] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n01:128830] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n01:128830] [ 4] [b19n11:133969] *** Process received signal *** +[b19n11:133969] Signal: Segmentation fault (11) +[b19n11:133969] Signal code: (3) +[b19n11:133969] Failing at address: 0xb170000000000a0 +[b19n11:133969] [ 0] [0x1000000504d8] +[b19n11:133969] [ 1] [0x3fffde5c6c50] +[b19n11:133969] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n11:133969] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n07:180192] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n07:180192] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n07:180192] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n07:180192] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n07:180192] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n11:160347] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n11:160347] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n11:160347] *** End of error message *** +[a27n01:48803] *** Process received signal *** +[a27n01:48803] Signal: Segmentation fault (11) +[a27n01:48803] Signal code: (3) +[a27n01:48803] Failing at address: 0x14170000000000a0 +[a27n01:48803] [ 0] [0x1000000504d8] +[a27n01:48803] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n13:114395] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n16:157885] [ 5] [0x1000000504d8] +[a36n10:142933] [ 1] [0x3fffdb61b700] +[a36n10:142933] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n10:142933] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n10:142933] [ 4] [0x1000000504d8] +[b21n16:116522] [ 1] [0x3fffc87ced70] +[b21n16:116522] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n16:116522] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n16:116522] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n12:167678] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n09:51333] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n09:51333] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n12:12764] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n12:12764] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n12:12764] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n12:12764] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n12:12764] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n06:126379] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n17:149907] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n17:149907] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n17:149907] [12] [0x1000000504d8] +[a29n15:177054] [ 1] [0x3fffdbb3b450] +[a29n15:177054] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n15:177054] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n15:177054] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n09:123418] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n09:123418] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n09:123418] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n09:123418] [14] [a31n15:155429] *** Process received signal *** +[a31n15:155429] Signal: Segmentation fault (11) +[a31n15:155429] Signal code: (3) +[a31n15:155429] Failing at address: 0x14170000000000a0 +[a31n15:155429] [ 0] [0x1000000504d8] +[a31n15:155429] [ 1] [0x3fffecc12920] +[a31n15:155429] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n15:155429] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n02:06562] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n02:06562] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n02:06562] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n02:06562] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n07:118163] [ 4] [b13n05:135163] *** Process received signal *** +[b13n05:135163] Signal: Segmentation fault (11) +[b13n05:135163] Signal code: (3) +[b13n05:135163] Failing at address: 0xb170000000000a0 +[b13n05:135163] [ 0] [b01n18:179696] [ 0] [0x1000000504d8] +[b01n18:179696] [ 1] [0x3fffe4a4aa00] +[b01n18:179696] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n18:179696] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n18:179696] [ 4] [b18n14:128867] *** Process received signal *** +[b18n14:128867] Signal: Segmentation fault (11) +[b18n14:128867] Signal code: (3) +[b18n14:128867] Failing at address: 0xb170000000000a0 +[b18n14:128867] [ 0] [0x1000000504d8] +[b18n14:128867] [ 1] [0x3fffc18a22e0] +[b18n14:128867] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n12:122964] *** End of error message *** +[b02n17:03843] [b16n14:133636] [ 0] [0x1000000504d8] +[b16n14:133636] [ 1] [0x3fffeaae5310] +[b16n14:133636] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b16n14:133636] [ 3] [b02n14:169520] *** Process received signal *** +[b02n14:169520] Signal: Segmentation fault (11) +[b02n14:169520] Signal code: (3) +[b02n14:169520] Failing at address: 0xb170000000000a0 +[b02n14:169520] [ 0] [0x1000000504d8] +[b02n14:169520] [ 1] [0x3ffff91a6e70] +[b02n14:169520] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n14:169520] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n13:122140] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n13:122140] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n13:122140] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n13:122140] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n13:122140] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n13:122140] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n13:122140] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n13:122140] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n13:122140] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n01:167298] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n01:167298] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n01:167298] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n01:167298] [14] [a27n17:15870] *** Process received signal *** +[a27n17:15870] Signal: Segmentation fault (11) +[a27n17:15870] Signal code: (3) +[a27n17:15870] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n13:176366] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n06:01840] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n06:01840] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n06:01840] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n01:141818] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n01:141818] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n01:141818] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n04:169912] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n04:169912] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n04:169912] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n07:138202] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n07:138202] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n07:138202] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n07:138202] *** End of error message *** +[a35n14:136659] [ 0] [0x1000000504d8] +[a35n14:136659] [ 1] [0x3fffd31f4ec0] +[a35n14:136659] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n14:136659] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n14:136659] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n10:176310] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n10:176310] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n10:176310] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n10:176310] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n08:08956] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n08:08956] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n08:08956] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n08:08956] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n06:143010] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n06:143010] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n06:143010] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n06:43018] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n06:43018] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n06:43018] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n06:43018] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n06:43018] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n14:159238] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n14:159238] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n14:159238] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n18:126903] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n18:126903] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n18:126903] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n18:126903] [14] [a33n08:147820] *** Process received signal *** +[a33n08:147820] Signal: Segmentation fault (11) +[a33n08:147820] Signal code: (3) +[a33n08:147820] Failing at address: 0xb170000000000a0 +[a33n08:147820] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n02:166746] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n02:166746] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n02:166746] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n02:166746] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n02:164488] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n02:164488] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n02:164488] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n02:164488] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n02:164488] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n07:132086] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n17:138141] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n07:131673] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n07:131673] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n07:131673] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n07:131673] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n07:131673] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n07:131673] [ 9] [a36n07:153175] *** Process received signal *** +[a36n07:153175] Signal: Segmentation fault (11) +[a36n07:153175] Signal code: (3) +[a36n07:153175] Failing at address: 0xb170000000000a0 +[a36n07:153175] [ 0] [0x1000000504d8] +[a36n07:153175] [ 1] [0x3fffc5ccce40] +[a36n07:153175] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n07:153175] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n07:153175] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n02:163028] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n02:163028] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n02:163028] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n02:163028] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n02:163028] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n17:150928] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n17:150928] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n17:150928] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n17:150928] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n17:150928] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n11:171964] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n11:171964] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n11:171964] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n11:171964] [13] [b02n16:01570] *** Process received signal *** +[b02n16:01570] Signal: Segmentation fault (11) +[b02n16:01570] Signal code: (3) +[b02n16:01570] Failing at address: 0xb170000000000a0 +[b02n16:01570] [ 0] [b18n01:128154] Signal: Segmentation fault (11) +[b18n01:128154] Signal code: (3) +[b18n01:128154] Failing at address: 0x14170000000000a0 +[b18n01:128154] [ 0] [0x1000000504d8] +[b18n01:128154] [ 1] [0x3ffff284cf40] +[b18n01:128154] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n01:128154] [ 3] [0x1000000504d8] +[a28n03:10289] [ 1] [0x3ffff51cb0c0] +[a28n03:10289] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n03:10289] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n03:10289] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n13:123709] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n13:123709] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n11:137874] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n11:137874] *** End of error message *** +[b15n09:121751] *** Process received signal *** +[b15n09:121751] Signal: Segmentation fault (11) +[b15n09:121751] Signal code: (3) +[b15n09:121751] Failing at address: 0xb170000000000a0 +[b15n09:121751] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n11:152985] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n11:152842] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n11:152842] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n11:152842] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n11:152842] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n11:152842] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n18:28217] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n18:28217] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n18:28217] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n18:28217] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n18:28217] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n11:01172] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n11:01172] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n11:01172] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n11:01172] [ 9] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n08:124723] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n08:124723] [ 4] [b01n09:173753] *** Process received signal *** +[b01n09:173753] Signal: Segmentation fault (11) +[b01n09:173753] Signal code: (3) +[b01n09:173753] Failing at address: 0x14170000000000a0 +[b01n09:173753] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n15:85943] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n15:85943] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n15:85943] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n08:174408] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n08:174408] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n13:01300] *** End of error message *** +[b21n01:130408] [ 0] [0x1000000504d8] +[b21n01:130408] [ 1] [0x3ffff0cdf4b0] +[b21n01:130408] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n01:130408] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n14:161618] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n14:161618] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n14:161618] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n14:161618] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n14:161618] [14] [0x3fffc0dc1490] +[b21n10:130579] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n10:130579] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n10:130579] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n10:130579] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n02:143966] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n02:143966] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n02:143966] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n16:153291] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n16:153291] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n16:153291] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n16:153291] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n16:122495] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n16:122495] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n06:122163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n06:122163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n06:122163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n06:122163] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n06:122163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n06:122163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n06:122163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n06:122163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n12:11145] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n12:11145] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n12:11145] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n12:11145] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n12:11145] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n09:119545] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n09:119545] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n12:148381] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n12:148381] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n12:148381] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n09:123811] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n09:123811] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n09:123811] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n16:171915] [ 5] [0x3fffc57c3090] +[a31n01:173781] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n01:173781] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n01:173781] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n01:173781] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n03:127628] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n03:127628] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n03:127628] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n12:156917] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n17:119582] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n02:120189] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n01:145734] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n01:145734] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n01:145734] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n01:145734] [14] [b06n04:175996] *** Process received signal *** +[b06n04:175996] Signal: Segmentation fault (11) +[b06n04:175996] Signal code: (3) +[b06n04:175996] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n15:148579] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n02:163322] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n04:138158] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n04:138158] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n04:138158] *** End of error message *** +[b10n17:164562] *** Process received signal *** +[b10n17:164562] Signal: Segmentation fault (11) +[b10n17:164562] Signal code: (3) +[b10n17:164562] Failing at address: 0xb170000000000a0 +[b10n17:164562] [ 0] [b18n10:122206] *** Process received signal *** +[b18n10:122206] Signal: Segmentation fault (11) +[b18n10:122206] Signal code: (3) +[b18n10:122206] Failing at address: 0xb170000000000a0 +[b18n10:122206] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n14:173896] *** End of error message *** +[a34n08:163163] *** Process received signal *** +[a34n08:163163] Signal: Segmentation fault (11) +[a34n08:163163] Signal code: (3) +[a34n08:163163] Failing at address: 0x14170000000000a0 +[a34n08:163163] [ 0] [a36n18:163109] *** Process received signal *** +[a36n18:163109] Signal: Segmentation fault (11) +[a36n18:163109] Signal code: (3) +[a36n18:163109] Failing at address: 0x14170000000000a0 +[a36n18:163109] [ 0] [0x1000000504d8] +[a36n18:163109] [ 1] [0x3ffff1c80f00] +[a36n18:163109] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n07:125474] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n07:125474] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n07:125474] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n07:125474] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n04:164046] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n04:164046] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n04:164046] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n04:164046] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n04:164046] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n04:164046] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n03:168208] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n03:168208] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n03:168208] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n03:168208] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n07:127355] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n07:127355] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n07:127355] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n11:147713] [ 4] [0x1000000504d8] +[b06n01:165162] [ 1] [0x3ffff12e8c10] +[b06n01:165162] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n01:165162] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n01:165162] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n01:165162] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n01:165162] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n08:156372] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n13:163195] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n13:165262] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n13:165262] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n13:165262] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n13:165262] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n13:165262] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n11:176298] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n11:176298] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n11:176298] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n11:176298] *** End of error message *** +[b23n12:115640] *** Process received signal *** +[b23n12:115640] Signal: Segmentation fault (11) +[b23n12:115640] Signal code: (3) +[b23n12:115640] Failing at address: 0x14170000000000a0 +[b23n12:115640] [ 0] [0x1000000504d8] +[b23n12:115640] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n02:155382] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n02:155382] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n02:155382] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n02:155382] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n02:155382] [ 8] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n04:169410] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n04:169410] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n08:163100] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n08:163100] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n08:163100] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n08:163100] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n08:163100] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n08:163100] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n08:163100] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n05:169785] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n05:169785] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n12:177120] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n12:177120] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n12:177120] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n08:121690] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n08:121690] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n15:154382] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n15:154382] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n05:148378] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n05:148378] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n03:39684] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n03:39684] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n03:39684] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n03:39684] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n03:39684] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n03:39684] *** End of error message *** +[b17n15:137005] *** Process received signal *** +[b17n15:137005] Signal: Segmentation fault (11) +[b17n15:137005] Signal code: (3) +[b17n15:137005] Failing at address: 0xb170000000000a0 +[b17n15:137005] [b03n11:175596] *** Process received signal *** +[b03n11:175596] Signal: Segmentation fault (11) +[b03n11:175596] Signal code: (3) +[b03n11:175596] Failing at address: 0x14170000000000a0 +[b03n11:175596] [ 0] [0x1000000504d8] +[b03n11:175596] [ 1] [0x3fffc20ed000] +[b03n11:175596] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n11:175596] [ 3] [b22n11:120238] *** Process received signal *** +[b22n11:120238] Signal: Segmentation fault (11) +[b22n11:120238] Signal code: (3) +[b22n11:120238] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n02:149226] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n02:149226] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n02:149226] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n02:149226] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n09:79007] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n11:152816] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n11:152816] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n11:152816] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n11:152816] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n11:152816] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n11:152816] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n01:138004] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n01:138004] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n01:138004] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n01:138004] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n01:138004] [14] [0x1000000504d8] +[b02n11:02683] [ 1] [0x3fffefeedfd0] +[b02n11:02683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n11:02683] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n17:146793] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n17:146793] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n17:146793] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n12:140121] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n12:140121] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n12:140121] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n12:140121] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n12:140121] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n12:140121] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n12:140121] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n12:140121] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n12:140121] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n06:10225] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n06:10225] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n06:10225] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n06:10225] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n06:10225] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n06:10225] [12] [b15n17:126524] [ 0] [0x1000000504d8] +[b15n17:126524] [ 1] [0x3fffe99c6ac0] +[b15n17:126524] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n17:126524] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n03:168932] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n03:168932] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n03:168932] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n03:168932] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n03:168932] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n09:164364] [ 6] [0x1000000504d8] +[b11n12:154345] [ 1] [0x3ffffa013f00] +[b11n12:154345] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n12:154345] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n12:154345] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n14:163549] [ 5] [b14n15:123037] [ 0] [0x1000000504d8] +[b14n15:123037] [ 1] [0x3fffdc9cf880] +[b14n15:123037] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n15:123037] [ 3] [b01n04:169706] *** Process received signal *** +[b01n04:169706] Signal: Segmentation fault (11) +[b01n04:169706] Signal code: (3) +[b01n04:169706] Failing at address: 0x5170000000000a0 +[b01n04:169706] [ 0] [0x1000000504d8] +[b01n04:169706] [ 1] [0x3fffd0c9d9f0] +[b01n04:169706] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n04:169706] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n04:169706] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n04:169706] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n12:11418] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n12:11418] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n12:11418] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n12:11418] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n12:11418] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n15:161756] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n15:161756] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n01:153683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n01:153683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n01:153683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n01:153683] *** End of error message *** +[b14n17:119464] *** Process received signal *** +[b14n17:119464] Signal: Segmentation fault (11) +[b14n17:119464] Signal code: (3) +[b14n17:119464] Failing at address: 0x14170000000000a0 +[b14n17:119464] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n06:177003] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n06:177003] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n06:177003] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n06:177003] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n16:124794] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n16:124794] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n01:65729] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n01:65729] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n01:65729] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n01:65729] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n01:65729] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n03:136667] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n03:136667] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n05:162566] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n05:162566] [ 9] [b23n03:125242] *** Process received signal *** +[b23n03:125242] Signal: Segmentation fault (11) +[b23n03:125242] Signal code: (3) +[b23n03:125242] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n11:148674] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n11:148674] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n11:148674] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n11:148674] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n11:148674] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n06:154623] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n06:154623] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n06:154623] [13] [a35n03:135161] *** Process received signal *** +[a35n03:135161] Signal: Segmentation fault (11) +[a35n03:135161] Signal code: (3) +[a35n03:135161] Failing at address: 0xb170000000000a0 +[a35n03:135161] [ 0] [0x1000000504d8] +[a35n03:135161] [ 1] [0x3fffc18238a0] +[a35n03:135161] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n03:135161] [ 3] [a33n09:147793] *** Process received signal *** +[a33n09:147793] Signal: Segmentation fault (11) +[a33n09:147793] Signal code: (3) +[a33n09:147793] Failing at address: 0x14170000000000a0 +[a33n09:147793] [ 0] [b01n03:01353] [ 0] [0x1000000504d8] +[b01n03:01353] [ 1] [0x3fffd5cfb950] +[b01n03:01353] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n03:01353] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n03:01353] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n02:122555] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n02:122555] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n02:122555] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n02:122555] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n02:178473] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n02:178473] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n02:127218] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n02:127218] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n02:127218] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n02:127218] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n10:161717] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n10:161717] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n10:161717] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n10:161717] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n02:123704] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n02:123704] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n09:163514] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n17:162357] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n17:162357] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n17:162357] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n17:162357] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n01:164964] [ 5] [0x1000000504d8] +[b20n06:130540] [ 1] [0x3ffff6a05b80] +[b20n06:130540] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n06:130540] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n06:130540] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n17:163818] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n17:163818] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n17:163818] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n17:163818] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n17:163818] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n17:163818] [ 9] [b18n06:133955] *** Process received signal *** +[b18n06:133955] Signal: Segmentation fault (11) +[b18n06:133955] Signal code: (3) +[b18n06:133955] Failing at address: 0xb170000000000a0 +[b19n05:131263] *** Process received signal *** +[b19n05:131263] Signal: Segmentation fault (11) +[b19n05:131263] Signal code: (3) +[b19n05:131263] Failing at address: 0x14170000000000a0 +[b19n05:131263] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n04:13079] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n04:13079] [ 6] [b18n05:135120] *** Process received signal *** +[b18n05:135120] Signal: Segmentation fault (11) +[b18n05:135120] Signal code: (3) +[b18n05:135120] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n03:151879] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n03:151879] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n03:151879] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n03:151879] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n01:179262] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n01:179262] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n02:147884] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n02:147884] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n08:175805] *** End of error message *** +[b01n08:172431] *** Process received signal *** +[b01n08:172431] Signal: Segmentation fault (11) +[b01n08:172431] Signal code: (3) +[b01n08:172431] Failing at address: 0x14170000000000a0 +[b01n08:172431] [ 0] [0x1000000504d8] +[b01n08:172431] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n05:54534] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n05:54534] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n05:54534] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n05:54534] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n05:54534] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n05:154188] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n05:154188] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n01:171786] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n01:171786] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n01:171786] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n01:171786] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n01:171786] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n04:142255] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n04:142255] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n04:142255] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n04:142255] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n04:142255] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n04:142255] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n17:118574] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n17:118574] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n17:118574] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n17:118574] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n01:141340] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n01:141340] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n03:153850] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n03:153850] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n03:153850] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n03:153850] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n05:130804] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n05:130804] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n05:130804] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n18:118681] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n18:118681] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n18:118681] [ 9] [b15n16:121930] [ 0] [0x1000000504d8] +[b15n16:121930] [ 1] [0x3fffc8d173f0] +[b15n16:121930] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n16:121930] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n16:121930] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n11:164993] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n11:164993] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n11:164993] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n11:164993] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n11:164993] [10] [a29n10:164936] *** Process received signal *** +[a29n10:164936] Signal: Segmentation fault (11) +[a29n10:164936] Signal code: (3) +[a29n10:164936] Failing at address: 0xb170000000000a0 +[a29n10:164936] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n08:164752] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n14:152037] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n14:152037] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n14:152037] [12] [b10n06:162915] Signal: Segmentation fault (11) +[b10n06:162915] Signal code: (3) +[b10n06:162915] Failing at address: 0xb170000000000a0 +[b10n06:162915] [ 0] [0x1000000504d8] +[b10n06:162915] [ 1] [0x3ffff2a1adf0] +[b10n06:162915] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n06:162915] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n01:132063] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n01:132063] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n01:132063] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n16:164549] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n16:164549] *** End of error message *** +[0x1000000504d8] +[b14n03:130349] [ 1] [0x3fffe838f360] +[b14n03:130349] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n03:130349] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n03:130349] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n02:116884] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n02:116884] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n02:116884] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n02:116884] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n02:116884] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n02:116884] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n02:116884] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n02:116884] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n02:116884] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n02:116884] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n02:116884] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n02:116884] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n06:171503] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n06:171503] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n06:171503] *** End of error message *** +[b15n06:120732] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n05:173080] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n05:173080] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n05:173080] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n07:03372] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n07:03372] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n02:139024] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n02:139024] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n02:139024] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n02:139024] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n02:139024] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n01:128005] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n01:128005] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n01:128005] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n01:128005] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n01:128005] *** End of error message *** +[b06n06:163674] *** Process received signal *** +[b06n06:163674] Signal: Segmentation fault (11) +[b06n06:163674] Signal code: (3) +[b06n06:163674] Failing at address: 0xb170000000000a0 +[b06n06:163674] [ 0] [0x1000000504d8] +[b06n06:163674] [ 1] [0x3fffe30aedb0] +[b06n06:163674] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n06:163674] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n11:160799] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n11:160799] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n11:160799] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n05:120333] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n05:120333] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n05:120333] *** End of error message *** +[0x3fffe8ce7ae0] +[b23n11:117473] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n11:117473] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n11:117473] [ 4] [b03n04:04848] *** Process received signal *** +[b03n04:04848] Signal: Segmentation fault (11) +[b03n04:04848] Signal code: (3) +[b03n04:04848] Failing at address: 0xb170000000000a0 +[b03n04:04848] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n17:123230] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n17:123230] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n17:123230] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n17:123230] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n17:123230] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n17:123230] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n17:123230] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n17:123230] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n17:123230] *** End of error message *** +[0x1000000504d8] +[b14n04:126858] [ 1] [0x3fffd5e44390] +[b14n04:126858] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n04:126858] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n04:126858] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n04:126858] [ 5] [b14n05:126778] *** Process received signal *** +[b14n05:126778] Signal: Segmentation fault (11) +[b14n05:126778] Signal code: (3) +[b14n05:126778] Failing at address: 0x14170000000000a0 +[b14n05:126778] [ 0] [0x1000000504d8] +[b14n05:126778] [ 1] [0x3fffe6e3d200] +[b14n05:126778] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n05:126778] [ 3] [b15n07:124349] *** Process received signal *** +[b15n07:124349] Signal: Segmentation fault (11) +[b15n07:124349] Signal code: (3) +[b15n07:124349] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n13:10195] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n13:10195] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n13:10195] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n13:10195] [ 9] [b10n01:165123] *** Process received signal *** +[b10n01:165123] Signal: Segmentation fault (11) +[b10n01:165123] Signal code: (3) +[b10n01:165123] Failing at address: 0xb170000000000a0 +[b10n01:165123] [ 0] [a32n09:155759] *** Process received signal *** +[a32n09:155759] Signal: Segmentation fault (11) +[a32n09:155759] Signal code: (3) +[a32n09:155759] Failing at address: 0x14170000000000a0 +[a32n09:155759] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n13:153677] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n13:153677] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n13:153677] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n13:153677] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n13:153677] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n06:157789] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n17:176412] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n17:176412] [ 5] [b12n12:138734] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n12:178600] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n12:178600] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n06:138762] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n06:138762] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n06:138762] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n06:138762] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n06:138762] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n06:138762] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n06:138762] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n06:138762] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n06:138762] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n06:138762] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n06:161724] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n06:161724] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n17:168154] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n17:168154] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n17:168154] [ 8] [b23n06:117528] *** Process received signal *** +[b23n06:117528] Signal: Segmentation fault (11) +[b23n06:117528] Signal code: (3) +[b23n06:117528] Failing at address: 0x14170000000000a0 +[b23n06:117528] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n10:16974] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n10:16974] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n10:16974] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n10:16974] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n10:16974] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n11:03984] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n11:03984] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n11:03984] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n11:03984] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n11:03984] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n11:03984] [13] [a29n07:160801] *** Process received signal *** +[a29n07:160801] Signal: Segmentation fault (11) +[a29n07:160801] Signal code: (3) +[a29n07:160801] Failing at address: 0x14170000000000a0 +[a29n07:160801] [ 0] [0x1000000504d8] +[a29n07:160801] [ 1] [0x3fffe21faaa0] +[a29n07:160801] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n07:160801] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n03:46691] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n03:46691] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n03:46691] *** End of error message *** +[b21n11:120583] *** Process received signal *** +[b21n11:120583] Signal: Segmentation fault (11) +[b21n11:120583] Signal code: (3) +[b21n11:120583] Failing at address: 0x14170000000000a0 +[b21n11:120583] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n04:158631] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n04:158631] [ 5] [b11n03:154293] *** Process received signal *** +[b11n03:154293] Signal: Segmentation fault (11) +[b11n03:154293] Signal code: (3) +[b11n03:154293] Failing at address: 0xb170000000000a0 +[b11n03:154293] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n14:178326] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n04:145749] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n04:145749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n04:145749] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n04:145749] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n04:145749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n04:145749] *** End of error message *** +[0x1000000504d8] +[b23n09:119632] [ 1] [0x3fffd398b720] +[b23n09:119632] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n09:119632] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n09:119632] [ 4] [0x1000000504d8] +[b02n13:171107] [ 1] [0x3fffcf5ab030] +[b02n13:171107] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n13:171107] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n13:171107] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n18:179169] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n18:179169] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n18:179169] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n18:179169] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n18:179169] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n13:123514] [ 4] [0x1000000504d8] +[a33n05:139436] [ 1] [0x3fffcc6a3c50] +[a33n05:139436] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n05:139436] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n05:139436] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n05:139436] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n05:139436] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n05:139436] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n11:134519] [ 6] [b08n07:168960] *** Process received signal *** +[b08n07:168960] Signal: Segmentation fault (11) +[b08n07:168960] Signal code: (3) +[b08n07:168960] Failing at address: 0x14170000000000a0 +[b08n07:168960] [ 0] [0x1000000504d8] +[b08n07:168960] [ 1] [0x3fffefe54b10] +[b08n10:11492] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n10:11492] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n10:11492] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n10:11492] [ 5] [b22n04:44758] *** Process received signal *** +[b22n04:44758] Signal: Segmentation fault (11) +[b22n04:44758] Signal code: (3) +[b22n04:44758] Failing at address: 0x14170000000000a0 +[b22n04:44758] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n17:169734] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n17:169734] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n17:169734] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n17:169734] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n17:169734] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n17:169734] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n03:06823] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n03:06823] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n03:06823] [ 6] [b14n12:123562] *** Process received signal *** +[b14n12:123562] Signal: Segmentation fault (11) +[b14n12:123562] Signal code: (3) +[b14n12:123562] Failing at address: 0xb170000000000a0 +[b14n12:123562] [ 0] [0x1000000504d8] +[b14n12:123562] [ 1] [0x3fffdda45c10] +[b14n12:123562] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n05:139912] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n05:156007] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n05:156007] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n05:156007] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n05:156007] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n05:156007] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n05:156007] *** End of error message *** +[a30n10:169209] *** Process received signal *** +[a30n10:169209] Signal: Segmentation fault (11) +[a30n10:169209] Signal code: (3) +[a30n10:169209] Failing at address: 0x14170000000000a0 +[a30n10:169209] [ 0] [0x1000000504d8] +[a30n10:169209] [ 1] [0x3fffcf5b1af0] +[a30n10:169209] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n10:169209] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n10:169209] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n10:169209] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n14:133970] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n14:133970] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n14:133970] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n14:133970] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n14:133970] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n01:163391] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n10:118390] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n10:118390] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n10:118390] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n05:174780] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n05:174780] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n05:174780] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n05:174780] [ 9] [b18n13:123052] *** Process received signal *** +[b18n13:123052] Signal: Segmentation fault (11) +[b18n13:123052] Signal code: (3) +[b18n13:123052] Failing at address: 0xb170000000000a0 +[b18n13:123052] [ 0] [0x1000000504d8] +[b18n13:123052] [ 1] [0x3fffe3b3b5b0] +[b18n13:123052] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n13:123052] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n13:154231] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n13:154231] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n13:154231] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n13:154231] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n13:154231] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n13:154231] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n13:154231] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n14:08399] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n14:08399] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n14:08399] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n08:141874] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n04:03148] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n04:03148] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n04:03148] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n04:03148] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n04:03148] [11] [b09n08:176682] *** Process received signal *** +[b09n08:176682] Signal: Segmentation fault (11) +[b09n08:176682] Signal code: (3) +[b09n08:176682] Failing at address: 0x14170000000000a0 +[b09n08:176682] [ 0] [0x1000000504d8] +[0x3ffff17a8700] +[b07n12:160963] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n12:160963] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n12:160963] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n12:160963] [ 5] [b04n01:96749] *** Process received signal *** +[b04n01:96749] Signal: Segmentation fault (11) +[b04n01:96749] Signal code: (3) +[b04n01:96749] Failing at address: 0x5170000000000a0 +[b04n01:96749] [ 0] [0x1000000504d8] +[b04n01:96749] [ 1] [0x3fffea47f090] +[b04n01:96749] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n12:145497] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n12:145497] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n12:145497] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n12:145497] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n12:145497] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n09:12246] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n09:12246] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n09:12246] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n09:12246] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n09:12246] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n11:130804] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n11:130804] [10] [b23n05:120227] Signal: Segmentation fault (11) +[b23n05:120227] Signal code: (3) +[b23n05:120227] Failing at address: 0x14170000000000a0 +[b23n05:120227] [ 0] [0x1000000504d8] +[b23n05:120227] [ 1] [0x3fffccb4ab80] +[b23n05:120227] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n05:120227] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n03:172649] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n03:172649] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n03:172649] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n03:172649] [ 6] [b13n13:133627] [ 0] [0x1000000504d8] +[b13n13:133627] [ 1] [0x3fffc5be0230] +[b13n13:133627] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n13:133627] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n14:152747] [ 7] [b18n15:120130] *** Process received signal *** +[b18n15:120130] Signal: Segmentation fault (11) +[b18n15:120130] Signal code: (3) +[b18n15:120130] Failing at address: 0xb170000000000a0 +[b18n15:120130] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n06:179453] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n06:179453] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n06:179453] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n06:179453] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n06:179453] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n06:179453] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n16:154183] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n16:154183] [10] [a33n10:140413] *** Process received signal *** +[a33n10:140413] Signal: Segmentation fault (11) +[a33n10:140413] Signal code: (3) +[a33n10:140413] Failing at address: 0x14170000000000a0 +[a33n10:140413] [ 0] [0x3fffeff229e0] +[a29n08:159024] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n08:159024] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n08:159024] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n08:159024] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n08:159024] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n08:159024] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n08:159024] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n08:159024] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n18:128543] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n18:128543] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n06:119115] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n06:119115] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n06:119115] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n06:119115] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n06:119115] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n06:119115] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n06:119115] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n06:119115] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n06:119115] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n06:119115] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n06:119115] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n11:156155] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n11:156155] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n11:156155] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n11:156155] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n02:122181] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n02:122181] [10] [b13n15:139749] *** Process received signal *** +[b13n15:139749] Signal: Segmentation fault (11) +[b13n15:139749] Signal code: (3) +[b13n15:139749] Failing at address: 0xb170000000000a0 +[b13n15:139749] [ 0] [0x1000000504d8] +[b13n15:139749] [ 1] [a28n05:04517] [ 0] [0x1000000504d8] +[a28n05:04517] [ 1] [0x3fffdffe92a0] +[a28n05:04517] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n05:04517] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n05:04517] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n05:04517] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n07:165092] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n07:165092] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n07:165092] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n07:165092] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n07:165092] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n15:122085] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n15:122085] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n15:122085] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n15:122085] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n15:122085] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n01:170768] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n01:170768] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n01:170768] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n01:170768] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n01:170768] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n10:10926] [ 4] [a02n09:160369] *** Process received signal *** +[a02n09:160369] Signal: Segmentation fault (11) +[a02n09:160369] Signal code: (3) +[a02n09:160369] Failing at address: 0xb170000000000a0 +[a02n09:160369] [ 0] [0x1000000504d8] +[a02n09:160369] [ 1] [0x3ffff4435970] +[a02n09:160369] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n09:160369] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n09:160369] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n09:160369] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n09:160369] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n09:160369] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n09:160369] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n09:160369] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n09:160369] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n09:160369] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n09:160369] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n09:160369] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n09:160369] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n09:160369] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n08:127445] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n08:127445] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n18:10723] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n09:168245] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n09:168245] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n10:155400] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n10:135851] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n10:135851] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n10:135851] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n10:135851] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n10:135851] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n10:135851] [ 9] [b09n10:169681] *** Process received signal *** +[b09n10:169681] Signal: Segmentation fault (11) +[b09n10:169681] Signal code: (3) +[b09n10:169681] Failing at address: 0xb170000000000a0 +[b09n10:169681] [ 0] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n02:74430] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n02:74430] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n02:74430] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n02:74430] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n02:74430] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n02:74430] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n07:172879] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n03:164495] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n03:164495] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n16:124960] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n16:124960] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n16:124960] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n16:124960] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n10:120978] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n10:120978] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n10:120978] [ 9] [b19n01:121746] *** Process received signal *** +[b19n01:121746] Signal: Segmentation fault (11) +[b19n01:121746] Signal code: (3) +[b19n01:121746] Failing at address: 0x14170000000000a0 +[b19n01:121746] [ 0] [0x1000000504d8] +[b19n01:121746] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n13:155850] *** End of error message *** +[b19n04:130745] *** Process received signal *** +[b19n04:130745] Signal: Segmentation fault (11) +[b19n04:130745] Signal code: (3) +[b19n04:130745] Failing at address: 0xb170000000000a0 +[b19n04:130745] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n12:160191] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n12:160191] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n12:122875] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n12:122875] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n12:122875] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n12:122875] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n07:45308] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n07:45308] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n07:45308] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n07:45308] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n07:45308] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n07:45308] [ 9] [0x3ffff63505a0] +[b18n02:125374] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n02:125374] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n02:125374] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n02:125374] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n02:125374] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n05:176911] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n05:176911] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n05:176911] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n05:176911] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n05:176911] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n09:106786] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n09:106786] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n09:106786] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n09:106786] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n09:106786] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n10:122756] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n10:122756] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n10:122756] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n09:170656] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n14:177512] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n14:177512] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n14:177512] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n08:146116] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n08:146116] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n08:146116] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n04:156413] *** End of error message *** +[0x1000000504d8] +[a31n07:156490] [ 1] [0x3fffc2383400] +[a31n07:156490] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n07:156490] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n03:135037] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n03:135037] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n03:135037] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n03:135037] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n09:128493] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n09:128493] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n09:128493] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n07:02840] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n07:02840] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n17:163177] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n17:163177] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n17:163177] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[0x1000000504d8] +[b06n10:176264] [ 1] [0x3fffdec6be20] +[b06n10:176264] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n10:176264] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n10:176264] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n10:176264] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n10:176264] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n10:176264] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n10:176264] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n10:176264] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n10:176264] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n10:176264] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n10:176264] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n10:176264] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n10:176264] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n10:176264] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n14:126119] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n14:126119] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n14:126119] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n14:126119] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n14:126119] [14] [b18n04:136994] *** Process received signal *** +[b18n04:136994] Signal: Segmentation fault (11) +[b18n04:136994] Signal code: (3) +[b18n04:136994] Failing at address: 0xb170000000000a0 +[b18n04:136994] [ 0] [0x1000000504d8] +[b18n04:136994] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n03:126187] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n03:126187] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n03:126187] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n03:126187] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n03:126187] *** End of error message *** +[0x1000000504d8] +[a33n03:147534] [ 1] [0x3fffeebe48f0] +[a33n03:147534] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n03:147534] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n03:147534] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n03:147534] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n08:118550] [ 5] [b15n11:134745] *** Process received signal *** +[b15n11:134745] Signal: Segmentation fault (11) +[b15n11:134745] Signal code: (3) +[b15n11:134745] Failing at address: 0x14170000000000a0 +[b15n11:134745] [ 0] [a26n03:68828] *** Process received signal *** +[a26n03:68828] Signal: Segmentation fault (11) +[a26n03:68828] Signal code: (3) +[a26n03:68828] Failing at address: 0xb170000000000a0 +[a26n03:68828] [ 0] [0x1000000504d8] +[a26n03:68828] [ 1] [0x3fffe1e8bb90] +[a26n03:68828] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n03:68828] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n03:68828] [ 4] [b14n14:122441] [ 0] [0x1000000504d8] +[b14n14:122441] [ 1] [0x3fffe9304ed0] +[b14n14:122441] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n14:122441] [ 3] [b09n05:173819] *** Process received signal *** +[b09n05:173819] Signal: Segmentation fault (11) +[b09n05:173819] Signal code: (3) +[b09n05:173819] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n01:123570] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n01:123570] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n01:123570] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n01:123570] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n12:145265] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n12:145265] [ 5] [a32n08:165284] *** Process received signal *** +[a32n08:165284] Signal: Segmentation fault (11) +[a32n08:165284] Signal code: (3) +[a32n08:165284] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n13:165386] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n13:165386] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n13:165386] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n13:165386] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n13:165386] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n13:165386] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n13:137748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n13:137748] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n03:119831] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n03:119831] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n03:119831] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n03:119831] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n06:148311] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n06:148311] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n06:148311] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n07:125451] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n07:125451] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n07:125451] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n04:136609] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n04:136609] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n04:136609] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n04:136609] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n04:136609] [11] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n08:124114] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n08:124114] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n08:124114] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n11:128953] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n11:128953] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n11:128953] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n11:128953] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n03:124681] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n07:163252] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n07:163252] [ 6] [b18n12:120404] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n05:05606] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n05:05606] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n05:05606] [ 6] [0x3fffcc352a70] +[b03n14:177222] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n14:177222] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n14:177222] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n14:177222] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n14:177222] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n14:177222] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n14:177222] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n14:177222] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n07:156140] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n07:156140] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n07:156140] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n18:121306] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n18:121306] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n18:121306] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n14:116410] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n16:129236] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n04:177597] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n04:177597] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n04:177597] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n13:177637] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n13:177637] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n13:177637] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n09:143896] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n14:139843] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n14:139843] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n14:139843] [ 8] [b23n04:117351] *** Process received signal *** +[b23n04:117351] Signal: Segmentation fault (11) +[b23n04:117351] Signal code: (3) +[b23n04:117351] Failing at address: 0xb170000000000a0 +[b23n04:117351] [ 0] [0x1000000504d8] +[b23n04:117351] [ 1] [0x3fffdd912540] +[b23n04:117351] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n04:117351] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n08:144855] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n08:138174] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n13:54242] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n13:54242] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n13:54242] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n13:54242] *** End of error message *** +[0x3fffe3b9d150] +[a28n12:177494] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n12:177494] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n12:177494] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n12:177494] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n12:177494] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n12:177494] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n12:177494] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n12:177494] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n12:177494] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n05:53373] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n05:53373] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n05:53373] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n05:53373] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n09:166574] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n09:166574] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n09:166574] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n12:180114] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n18:122553] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n18:122553] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n07:149476] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n07:149476] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n07:149476] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n07:149476] [10] [0x3fffd39b1300] +[b20n05:121822] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n05:121822] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n05:121822] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n05:121822] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n05:121822] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n16:130377] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n16:130377] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n16:130377] [ 8] [b10n10:126191] *** Process received signal *** +[b10n10:126191] Signal: Segmentation fault (11) +[b10n10:126191] Signal code: (3) +[b10n10:126191] Failing at address: 0x5170000000000a0 +[b10n10:126191] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n15:148167] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n16:147287] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n16:147287] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n16:147287] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n16:147287] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n16:147287] *** End of error message *** +[a27n15:32398] [ 0] [0x1000000504d8] +[a27n15:32398] [ 1] [0x3fffc7fa9590] +[a27n15:32398] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n15:32398] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n18:03312] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n18:03312] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n04:168489] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n04:168489] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n04:168489] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n04:168489] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n04:168489] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n13:138090] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n13:138090] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n12:164382] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n12:164382] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n12:164382] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n12:164382] [ 9] [b17n17:125805] [ 0] [0x1000000504d8] +[b17n17:125805] [ 1] [0x3fffeb445140] +[b17n17:125805] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n17:125805] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n14:136135] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n14:136135] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n11:124446] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n11:124446] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n11:124446] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n11:124446] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n15:119254] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n15:119254] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n15:119254] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n15:119254] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n15:120061] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n01:163023] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n01:163023] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n01:163023] [ 8] [a36n15:151248] *** Process received signal *** +[a36n15:151248] Signal: Segmentation fault (11) +[a36n15:151248] Signal code: (3) +[a36n15:151248] Failing at address: 0x5170000000000a0 +[a36n15:151248] [ 0] [0x1000000504d8] +[a36n15:151248] [ 1] [0x3fffc8cb39f0] +[a36n15:151248] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n11:163260] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n11:163260] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n11:163260] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n11:163260] [ 8] [a34n14:150131] [ 0] [0x1000000504d8] +[a34n14:150131] [ 1] [0x3fffcbfbb0e0] +[a34n14:150131] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n14:150131] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n12:178717] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n04:153962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n04:153962] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n04:153962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n04:153962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n10:134733] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n10:134733] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n10:134733] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n13:117508] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n13:117508] [ 6] [b21n14:119487] *** Process received signal *** +[b21n14:119487] Signal: Segmentation fault (11) +[b21n14:119487] Signal code: (3) +[b21n14:119487] Failing at address: 0x14170000000000a0 +[b21n14:119487] [ 0] [0x1000000504d8] +[b21n14:119487] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n17:168160] *** End of error message *** +[b20n04:126348] *** Process received signal *** +[b20n04:126348] Signal: Segmentation fault (11) +[b20n04:126348] Signal code: (3) +[b20n04:126348] Failing at address: 0x14170000000000a0 +[b20n04:126348] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n15:124801] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n15:124801] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n15:124801] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n11:162470] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n11:162470] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n02:119534] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n03:158655] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n03:158655] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n03:158655] *** End of error message *** +[a33n13:168158] [ 0] [0x1000000504d8] +[a33n13:168158] [ 1] [0x3fffe516e5d0] +[a33n13:168158] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n13:168158] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n11:153415] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n11:153415] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n11:153415] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n11:124432] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n16:173550] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n16:173550] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n16:173550] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n16:149062] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n05:125612] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n05:125612] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n05:125612] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n05:125612] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n05:125612] [13] [b11n09:161228] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n18:140386] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n18:140386] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n18:140386] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n18:140386] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n18:140386] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n18:140386] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n18:140386] [10] [b09n18:164799] *** Process received signal *** +[b09n18:164799] Signal: Segmentation fault (11) +[b09n18:164799] Signal code: (3) +[b09n18:164799] Failing at address: 0xb170000000000a0 +[b09n18:164799] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n13:166924] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n13:166924] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n13:166924] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n18:08427] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n12:120128] [ 4] [b13n17:129929] [ 0] [0x1000000504d8] +[b13n17:129929] [ 1] [0x3ffff859ef20] +[b13n17:129929] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n17:129929] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n17:129929] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n16:128439] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n16:128439] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n16:128439] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n16:128439] [13] [b19n13:123884] *** Process received signal *** +[b19n13:123884] Signal: Segmentation fault (11) +[b19n13:123884] Signal code: (3) +[b19n13:123884] Failing at address: 0xb170000000000a0 +[b19n13:123884] [ 0] [0x1000000504d8] +[b19n13:123884] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n05:164944] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n05:164944] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n05:164944] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n05:164944] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n08:54626] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n18:131584] [10] [a32n10:162608] *** Process received signal *** +[a32n10:162608] Signal: Segmentation fault (11) +[a32n10:162608] Signal code: (3) +[a32n10:162608] Failing at address: 0xb170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n17:171199] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n17:171199] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n17:171199] *** End of error message *** +[b19n14:133063] *** Process received signal *** +[b19n14:133063] Signal: Segmentation fault (11) +[b19n14:133063] Signal code: (3) +[b19n14:133063] Failing at address: 0x14170000000000a0 +[b19n14:133063] [ 0] [0x1000000504d8] +[b19n14:133063] [ 1] [0x3fffe52808a0] +[b19n14:133063] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n14:133063] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n14:133063] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n02:167598] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[0x1000000504d8] +[b11n04:166205] [ 1] [0x3ffff1acb890] +[b11n04:166205] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n04:166205] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n04:166205] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n04:166205] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n08:131380] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n08:131380] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n08:131380] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n08:131380] [ 9] [b21n08:116175] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n06:127276] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n06:127276] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n16:165925] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n17:124163] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n04:52707] *** End of error message *** +[b21n07:117358] [ 0] [0x1000000504d8] +[b21n07:117358] [ 1] [0x3fffc063d490] +[b21n07:117358] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n07:117358] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n07:133373] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n02:176050] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n12:132099] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n04:41672] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n04:41672] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n04:41672] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n04:41672] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n18:07421] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n18:07421] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n16:127353] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n11:23061] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n11:23061] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n11:23061] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n11:23061] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n11:23061] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n11:23061] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n11:23061] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n11:23061] *** End of error message *** +[a31n06:167370] *** Process received signal *** +[a31n06:167370] Signal: Segmentation fault (11) +[a31n06:167370] Signal code: (3) +[a31n06:167370] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n14:168375] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n14:168375] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n14:168375] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n14:168375] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n14:168375] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n14:176069] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n13:122030] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n03:123614] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n03:123614] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n03:123614] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n03:123614] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n03:38503] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n03:38503] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n03:38503] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n03:38503] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n03:38503] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n03:38503] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n11:170894] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n11:170894] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n11:170894] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n11:170894] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n11:170894] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n11:170894] [ 8] [0x3fffc5d111d0] +[b20n01:128831] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n01:128831] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n01:128831] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n01:128831] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n01:128831] [ 6] [b19n11:133971] *** Process received signal *** +[b19n11:133971] Signal: Segmentation fault (11) +[b19n11:133971] Signal code: (3) +[b19n11:133971] Failing at address: 0x14170000000000a0 +[b19n11:133971] [ 0] [0x1000000504d8] +[b19n11:133971] [ 1] [0x3fffe100d8b0] +[b19n11:133971] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n11:133971] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n11:133971] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n11:133971] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n11:133971] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n11:133971] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n11:133971] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n11:133971] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n11:133971] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n11:133971] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n11:133971] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n11:133971] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n11:133971] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n11:133971] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n07:180192] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n11:160348] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n11:160348] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n16:157885] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n16:157885] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n13:114395] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n13:114395] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n13:114395] [ 8] [0x1000000504d8] +[a36n10:142932] [ 1] [0x3fffde93c890] +[a36n10:142932] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n10:142932] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n10:142932] [ 4] [0x1000000504d8] +[b21n16:116523] [ 1] [0x3fffcb65dc60] +[b21n16:116523] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n16:116523] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n16:116523] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n12:12765] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n12:12765] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n12:12765] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n12:12765] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n12:12765] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n01:48802] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n12:167678] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n09:51334] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n09:51334] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n09:51334] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n09:51334] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n17:149907] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n17:149907] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n17:149907] *** End of error message *** +[0x1000000504d8] +[a29n15:177055] [ 1] [0x3fffd737adc0] +[a29n15:177055] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n15:177055] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n15:177055] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n09:123419] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n09:123419] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n09:123419] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n06:126379] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n15:155430] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n15:155430] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n15:155430] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n15:155430] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n15:155430] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n15:155430] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n15:155430] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n15:155430] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n15:155430] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n02:06562] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n02:06562] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n07:118163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n07:118163] [ 6] [b01n18:179697] *** Process received signal *** +[b01n18:179697] Signal: Segmentation fault (11) +[b01n18:179697] Signal code: (3) +[b01n18:179697] Failing at address: 0xb170000000000a0 +[b01n18:179697] [ 0] [0x1000000504d8] +[b01n18:179697] [ 1] [0x3fffde9d34e0] +[b01n18:179697] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n18:179697] [ 3] [b13n05:135162] *** Process received signal *** +[b13n05:135162] Signal: Segmentation fault (11) +[b13n05:135162] Signal code: (3) +[b13n05:135162] Failing at address: 0x14170000000000a0 +[b13n05:135162] [ 0] [0x1000000504d8] +[b13n05:135162] [ 1] [0x3fffe50e1b20] +[b13n05:135162] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n05:135162] [ 3] [0x1000000504d8] +[b18n14:128868] [ 1] [0x3ffff82f09f0] +[b18n14:128868] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n14:128868] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n14:128868] [ 4] [b22n12:122966] *** Process received signal *** +[b22n12:122966] Signal: Segmentation fault (11) +[b22n12:122966] Signal code: (3) +[b22n12:122966] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n14:133635] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n14:133635] [ 6] [ 6] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n14:169521] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n14:169521] [ 7] [0x3ffffd37c0d0] +[b14n13:122143] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n13:122143] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n13:122143] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n13:122143] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n13:122143] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n13:122143] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n13:122143] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n13:122143] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n13:122143] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n13:122143] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n13:122143] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n13:122143] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n13:122143] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n13:122143] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n01:167297] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n01:167297] *** End of error message *** +[a27n17:15871] *** Process received signal *** +[a27n17:15871] Signal: Segmentation fault (11) +[a27n17:15871] Signal code: (3) +[a27n17:15871] Failing at address: 0x14170000000000a0 +[a27n17:15871] [ 0] [0x1000000504d8] +[a27n17:15871] [ 1] [0x3fffd4cbefd0] +[a27n17:15871] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n17:15871] [ 3] [a28n13:176364] *** Process received signal *** +[a28n13:176364] Signal: Segmentation fault (11) +[a28n13:176364] Signal code: (3) +[a28n13:176364] Failing at address: 0xb170000000000a0 +[a35n14:136658] *** Process received signal *** +[a35n14:136658] Signal: Segmentation fault (11) +[a35n14:136658] Signal code: (3) +[a35n14:136658] Failing at address: 0xb170000000000a0 +[a35n14:136658] [ 0] [0x1000000504d8] +[a35n14:136658] [ 1] [0x3fffe37f2c10] +[a35n14:136658] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n14:136658] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n14:136658] [ 4] [0x3fffd7b12370] +[b03n06:01839] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n06:01839] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n06:01839] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n06:01839] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n06:01839] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n01:141819] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n10:176309] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n10:176309] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n10:176309] *** End of error message *** +[a35n07:138204] *** Process received signal *** +[a35n07:138204] Signal: Segmentation fault (11) +[a35n07:138204] Signal code: (3) +[a35n07:138204] Failing at address: 0x5170000000000a0 +[a35n07:138204] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n08:08955] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n08:08955] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n08:08955] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n08:08955] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n06:143009] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n06:143009] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n06:143009] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n18:126904] *** End of error message *** +[b20n07:132087] *** Process received signal *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n02:164489] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n02:164489] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n02:164489] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n02:164489] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n04:169912] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n04:169912] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n02:166747] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n02:166747] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n02:166747] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n02:166747] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n07:153174] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n07:153174] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n07:153174] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n07:153174] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n07:153174] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n07:153174] [10] [0x1000000504d8] +[a33n08:147820] [ 1] [0x3fffdd61d600] +[a33n08:147820] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n08:147820] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n07:131672] *** End of error message *** +[b02n16:01569] [ 0] [0x1000000504d8] +[b02n16:01569] [ 1] [0x3fffe65744a0] +[b02n16:01569] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n16:01569] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n16:01569] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n17:150927] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n17:150927] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n17:150927] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n17:150927] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n01:128154] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n11:171965] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n11:171965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n11:171965] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n11:171965] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n11:171965] *** End of error message *** +[b12n11:137869] *** Process received signal *** +[b12n11:137869] Signal: Segmentation fault (11) +[b12n11:137869] Signal code: (3) +[b12n11:137869] Failing at address: 0x14170000000000a0 +[b15n13:123706] *** Process received signal *** +[0x1000000504d8] +[a28n03:10288] [ 1] [0x3fffda266670] +[a28n03:10288] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n03:10288] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n03:10288] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n03:10288] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n11:152985] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n09:121752] *** Process received signal *** +[b15n09:121752] Signal: Segmentation fault (11) +[b15n09:121752] Signal code: (3) +[b15n09:121752] Failing at address: 0x14170000000000a0 +[b15n09:121752] [ 0] [0x1000000504d8] +[b15n09:121752] [ 1] [0x3fffc921f4c0] +[b15n09:121752] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n09:121752] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n11:152841] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n18:28216] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n18:28216] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n08:124722] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n08:124722] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n08:124722] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n08:124722] [ 9] [b01n09:173752] *** Process received signal *** +[b01n09:173752] Signal: Segmentation fault (11) +[b01n09:173752] Signal code: (3) +[b01n09:173752] Failing at address: 0x14170000000000a0 +[b01n09:173752] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n15:85943] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n15:85943] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n11:01171] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n11:01171] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n11:01171] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n08:174409] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n08:174409] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n08:174409] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n16:171916] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n16:171916] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n09:123811] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n09:123811] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n09:123811] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n09:123811] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n09:123811] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n14:161618] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n02:143966] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n02:143966] *** End of error message *** +[b14n16:122491] *** Process received signal *** +[b14n16:122491] Signal: Segmentation fault (11) +[b14n16:122491] Signal code: (3) +[b14n16:122491] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n16:153292] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n16:153292] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n06:122162] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n06:122162] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n06:122162] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n06:122162] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n06:122162] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n09:119546] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n09:119546] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n09:119546] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n09:119546] [ 8] [b22n17:119580] *** Process received signal *** +[b22n17:119580] Signal: Segmentation fault (11) +[b22n17:119580] Signal code: (3) +[b22n17:119580] Failing at address: 0xb170000000000a0 +[b22n17:119580] [ 0] [0x1000000504d8] +[b22n17:119580] [ 1] [0x3ffffe18ce00] +[b22n17:119580] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n17:119580] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n17:119580] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n17:119580] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n17:119580] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n12:148382] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n12:148382] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n12:148382] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n12:148382] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n12:148382] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n12:148382] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n01:173782] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n01:173782] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n01:173782] [ 8] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n03:127629] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n03:127629] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n03:127629] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n01:130408] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n01:130408] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n02:120188] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n02:120188] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n10:130580] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n10:130580] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n10:130580] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n10:130580] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n12:156917] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n01:145733] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n01:145733] *** End of error message *** +[b06n04:175997] *** Process received signal *** +[b06n04:175997] Signal: Segmentation fault (11) +[b06n04:175997] Signal code: (3) +[b06n04:175997] Failing at address: 0xb170000000000a0 +[a33n15:148582] *** Process received signal *** +[a33n15:148582] Signal: Segmentation fault (11) +[a33n15:148582] Signal code: (3) +[a33n15:148582] Failing at address: 0x5170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n04:138159] *** End of error message *** +[b06n02:163318] *** Process received signal *** +[b06n02:163318] Signal: Segmentation fault (11) +[b06n02:163318] Signal code: (3) +[b06n02:163318] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n12:11144] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n12:11144] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n12:11144] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n12:11144] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n12:11144] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n12:11144] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n12:11144] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n12:11144] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n12:11144] *** End of error message *** +[b23n12:115639] Signal: Segmentation fault (11) +[b23n12:115639] Signal code: (3) +[b23n12:115639] Failing at address: 0x14170000000000a0 +[b23n12:115639] [ 0] [0x1000000504d8] +[b23n12:115639] [ 1] [0x3ffff766fcf0] +[b23n12:115639] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n12:115639] [ 3] [0x1000000504d8] +[b10n17:164562] [ 1] [0x3fffdf3c6aa0] +[b10n17:164562] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n17:164562] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n17:164562] [ 4] [0x1000000504d8] +[b18n10:122205] [ 1] [0x3ffffcf27d70] +[b18n10:122205] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n10:122205] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n10:122205] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n10:122205] [ 5] [a34n08:163164] *** Process received signal *** +[a34n08:163164] Signal: Segmentation fault (11) +[a34n08:163164] Signal code: (3) +[a34n08:163164] Failing at address: 0x14170000000000a0 +[a34n08:163164] [ 0] [0x1000000504d8] +[a34n08:163164] [ 1] [0x3fffdd78ef30] +[a34n08:163164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n08:163164] [ 3] [0x3fffd71a6ef0] +[a36n18:163110] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n18:163110] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n18:163110] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n18:163110] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n18:163110] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n18:163110] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n18:163110] [ 8] [b17n07:125470] *** Process received signal *** +[b17n07:125470] Signal: Segmentation fault (11) +[b17n07:125470] Signal code: (3) +[b17n07:125470] Failing at address: 0x14170000000000a0 +[b17n07:125470] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n03:168209] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n03:168209] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n03:168209] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n03:168209] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n03:168209] [10] [b14n07:127357] *** Process received signal *** +[b14n07:127357] Signal: Segmentation fault (11) +[b14n07:127357] Signal code: (3) +[b14n07:127357] Failing at address: 0xb170000000000a0 +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n04:164047] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n04:164047] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n04:164047] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n04:164047] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n11:147713] [ 5] [b06n14:173892] *** Process received signal *** +[b06n14:173892] Signal: Segmentation fault (11) +[b06n14:173892] Signal code: (3) +[b06n14:173892] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n01:165163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n01:165163] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n01:165163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n01:165163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n01:165163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n01:165163] [10] [a31n08:156368] *** Process received signal *** +[a31n08:156368] Signal: Segmentation fault (11) +[a31n08:156368] Signal code: (3) +[a31n08:156368] Failing at address: 0xb170000000000a0 +[a31n08:156368] [ 0] [0x1000000504d8] +[a31n08:156368] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n13:163194] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n13:165261] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n13:165261] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n13:165261] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n13:165261] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n13:165261] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n04:169409] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n04:169409] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n04:169409] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n04:169409] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n08:163099] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n08:163099] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n08:163099] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n08:163099] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n11:176297] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n11:176297] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n11:176297] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n11:176297] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n11:176297] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n02:155382] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n02:155382] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n02:155382] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n02:155382] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n02:155382] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n02:155382] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n12:177119] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n12:177119] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n12:177119] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n05:169785] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n05:169785] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n15:154383] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n15:154383] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n15:154383] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n15:154383] [ 9] [b19n08:121686] *** Process received signal *** +[b19n08:121686] Signal: Segmentation fault (11) +[b19n08:121686] Signal code: (3) +[b19n08:121686] Failing at address: 0x14170000000000a0 +[b19n08:121686] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n03:39683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n03:39683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n03:39683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n03:39683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n03:39683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n03:39683] *** End of error message *** +[b17n15:137004] *** Process received signal *** +[b17n15:137004] Signal: Segmentation fault (11) +[b17n15:137004] Signal code: (3) +[b17n15:137004] Failing at address: 0x14170000000000a0 +[b17n15:137004] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n11:175596] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n11:175596] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n11:175596] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n11:175596] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n11:175596] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n11:175596] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n11:175596] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n11:175596] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n11:175596] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n11:175596] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n11:175596] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n11:175596] *** End of error message *** +[b22n11:120238] [ 0] [0x1000000504d8] +[b22n11:120238] [ 1] [0x3fffdba01bd0] +[b22n11:120238] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n11:120238] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n09:79006] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n09:79006] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n02:149227] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n02:149227] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n02:149227] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n11:152817] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n11:152817] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n11:152817] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n11:152817] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n11:152817] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n11:152817] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n11:152817] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n11:152817] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n01:138003] *** End of error message *** +[b02n11:02682] *** Process received signal *** +[b02n11:02682] Signal: Segmentation fault (11) +[b02n11:02682] Signal code: (3) +[b02n11:02682] Failing at address: 0x14170000000000a0 +[b02n11:02682] [ 0] [0x1000000504d8] +[b02n11:02682] [ 1] [0x3ffff06b28a0] +[b02n11:02682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n11:02682] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n17:146794] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n17:146794] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n17:146794] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n17:146794] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n17:146794] *** End of error message *** +[b13n12:140124] *** Process received signal *** +[b13n12:140124] Signal: Segmentation fault (11) +[b13n12:140124] Signal code: (3) +[b13n12:140124] Failing at address: 0xb170000000000a0 +[b13n12:140124] [ 0] [0x1000000504d8] +[b13n12:140124] [ 1] [0x3fffed86a6b0] +[b13n12:140124] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n12:140124] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n12:140124] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n12:140124] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n12:140124] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n17:126524] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n06:10226] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n06:10226] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n06:10226] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n06:10226] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n06:10226] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n06:10226] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n06:10226] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n06:10226] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n03:168932] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n09:164364] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n09:164364] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n14:163549] [ 6] [b14n15:123038] *** Process received signal *** +[b14n15:123038] Signal: Segmentation fault (11) +[b14n15:123038] Signal code: (3) +[b14n15:123038] Failing at address: 0xb170000000000a0 +[b14n15:123038] [ 0] [0x1000000504d8] +[b14n15:123038] [ 1] [0x3fffe1d0da00] +[b14n15:123038] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n15:123038] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n03:135161] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n03:135161] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n12:154345] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n12:11418] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n12:11418] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n12:11418] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n15:161757] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n15:161757] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n15:161757] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n16:124793] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n16:124793] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n16:124793] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n16:124793] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n16:124793] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n16:124793] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n16:124793] *** End of error message *** +[0x1000000504d8] +[b14n17:119464] [ 1] [0x3fffdab038e0] +[b14n17:119464] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n17:119464] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n06:177004] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n06:177004] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n03:136668] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n03:136668] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n03:136668] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n03:136668] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n03:136668] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n01:65726] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n01:65726] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n01:65726] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n01:65726] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n01:65726] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n01:65726] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n01:65726] *** End of error message *** +[b23n03:125243] *** Process received signal *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n06:154623] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n06:154623] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n04:169705] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n05:162566] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n05:162566] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n05:162566] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n05:162566] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n05:162566] [14] [a33n09:147794] *** Process received signal *** +[a33n09:147794] Signal: Segmentation fault (11) +[a33n09:147794] Signal code: (3) +[a33n09:147794] Failing at address: 0x14170000000000a0 +[a33n09:147794] [ 0] [0x1000000504d8] +[a33n09:147794] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n11:148675] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n11:148675] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n03:01354] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n03:01354] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n03:01354] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n03:01354] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n02:178475] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n02:178475] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n02:178475] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n02:178475] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n02:178475] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n02:178475] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n02:122555] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n02:122555] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n02:122555] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n02:127219] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n02:127219] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n02:127219] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n02:127219] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n02:127219] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n02:127219] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n10:161716] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n10:161716] [b19n02:123702] *** Process received signal *** +[b19n02:123702] Signal: Segmentation fault (11) +[b19n02:123702] Signal code: (3) +[b19n02:123702] Failing at address: 0xb170000000000a0 +[b19n02:123702] [ 0] [0x1000000504d8] +[b19n02:123702] [ 1] [0x3fffe133d960] +[b19n02:123702] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n17:162358] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n17:162358] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n17:162358] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n17:162358] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n01:164964] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n01:164964] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n06:130541] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n06:130541] [ 5] [ 5] [b18n06:133955] [ 0] [0x1000000504d8] +[b18n06:133955] [ 1] [0x3fffc7fa7040] +[b18n06:133955] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n06:133955] [ 3] [0x1000000504d8] +[b19n05:131263] [ 1] [0x3fffc7019bd0] +[b19n05:131263] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n05:131263] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n04:13080] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n04:13080] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n04:13080] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n03:151880] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n03:151880] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n03:151880] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n03:151880] [12] [b18n05:135119] [ 0] [0x1000000504d8] +[b18n05:135119] [ 1] [0x3fffcaf580f0] +[b18n05:135119] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n05:135119] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n05:135119] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n01:179263] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n01:179263] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n02:147885] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n02:147885] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n05:154189] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n05:154189] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n05:154189] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n05:154189] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n05:154189] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n05:154189] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n01:171785] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n01:171785] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n01:171785] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n01:171785] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n01:171785] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n17:118574] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n17:118574] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n04:142254] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n04:142254] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n04:142254] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n04:142254] [10] [b01n08:172429] *** Process received signal *** +[b01n08:172429] Signal: Segmentation fault (11) +[b01n08:172429] Signal code: (3) +[b01n08:172429] Failing at address: 0x14170000000000a0 +[b01n08:172429] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n01:141341] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n03:153850] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n03:153850] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n03:153850] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n03:153850] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n03:153850] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n06:171500] *** Process received signal *** +[b01n06:171500] Signal: Segmentation fault (11) +[b01n06:171500] Signal code: (3) +[b01n06:171500] Failing at address: 0xb170000000000a0 +[b01n06:171500] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n16:121929] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n16:121929] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n16:121929] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n11:164994] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n11:164994] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n11:164994] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n11:164994] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n11:164994] [10] [b08n08:164749] *** Process received signal *** +[b08n08:164749] Signal: Segmentation fault (11) +[b08n08:164749] Signal code: (3) +[b08n08:164749] Failing at address: 0xb170000000000a0 +[b08n08:164749] [ 0] [0x1000000504d8] +[b08n08:164749] [ 1] [0x3fffdd40db00] +[b08n08:164749] [ 2] [0x1000000504d8] +[a29n10:164937] [ 1] [0x3fffe141b3a0] +[a29n10:164937] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n10:164937] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n10:164937] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n10:164937] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n10:164937] [ 6] [0x1000000504d8] +[b10n06:162916] [ 1] [0x3fffc1159750] +[b10n06:162916] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n06:162916] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n06:162916] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n14:152036] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n14:152036] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n14:152036] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n14:152036] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n14:152036] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n01:132062] *** End of error message *** +[0x1000000504d8] +[b03n04:04848] [ 1] [0x3ffffe2708d0] +[b03n04:04848] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n04:04848] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n04:04848] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n03:130349] [ 5] [b23n02:116885] *** Process received signal *** +[b23n02:116885] Signal: Segmentation fault (11) +[b23n02:116885] Signal code: (3) +[b23n02:116885] Failing at address: 0xb170000000000a0 +[b23n02:116885] [ 0] [0x1000000504d8] +[b23n02:116885] [ 1] [0x3fffdce05770] +[b23n02:116885] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n02:116885] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n02:116885] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n02:116885] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n02:116885] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n02:116885] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n02:116885] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n02:116885] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n02:116885] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n02:116885] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n02:116885] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n02:116885] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n02:116885] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n02:116885] *** End of error message *** +[b15n06:120732] Signal: Segmentation fault (11) +[b15n06:120732] Signal code: (3) +[b15n06:120732] Failing at address: 0xb170000000000a0 +[b15n06:120732] [ 0] [0x1000000504d8] +[b15n06:120732] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n18:118681] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n18:118681] [11] [0x1000000504d8] +[b15n07:124348] [ 1] [0x3fffcc332480] +[b15n07:124348] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n07:124348] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n07:124348] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n07:03371] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n07:03371] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n07:03371] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n07:03371] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n07:03371] *** End of error message *** +[b06n06:163675] [ 0] [0x1000000504d8] +[b06n06:163675] [ 1] [0x3fffde9cd4a0] +[b06n06:163675] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n06:163675] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n06:163675] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n05:173080] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n05:173080] *** End of error message *** +[b14n01:128004] *** End of error message *** +[b06n11:160795] *** Process received signal *** +[b06n11:160795] Signal: Segmentation fault (11) +[b06n11:160795] Signal code: (3) +[b06n11:160795] Failing at address: 0x5170000000000a0 +[b06n11:160795] [ 0] [0x1000000504d8] +[b12n12:138735] [ 1] [0x3fffff7d1c30] +[b12n12:138735] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n12:138735] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n12:138735] [ 4] [b15n05:120329] *** Process received signal *** +[b15n05:120329] Signal: Segmentation fault (11) +[b15n05:120329] Signal code: (3) +[b15n05:120329] Failing at address: 0xb170000000000a0 +[b15n05:120329] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n17:123234] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n17:123234] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n17:123234] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n17:123234] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n17:123234] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n17:123234] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n04:126857] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n04:126857] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n16:164550] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n16:164550] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n16:164550] [ 8] [0x1000000504d8] +[b14n05:126779] [ 1] [0x3fffde9f22f0] +[b14n05:126779] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n05:126779] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n05:126779] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n05:126779] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n13:10196] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n13:10196] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n13:10196] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n13:10196] [ 9] [0x1000000504d8] +[b10n01:165122] [ 1] [0x3fffc12bdd30] +[b10n01:165122] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n01:165122] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n01:165122] [ 4] [b14n02:139027] *** Process received signal *** +[b14n02:139027] Signal: Segmentation fault (11) +[b14n02:139027] Signal code: (3) +[b14n02:139027] Failing at address: 0x14170000000000a0 +[b14n02:139027] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n09:155760] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n09:155760] [ 5] [a29n06:157787] *** Process received signal *** +[a29n06:157787] Signal: Segmentation fault (11) +[a29n06:157787] Signal code: (3) +[a29n06:157787] Failing at address: 0xb170000000000a0 +[a29n06:157787] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n13:153675] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n13:153675] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n13:153675] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n13:153675] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n11:117473] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n11:117473] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n12:178599] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n12:178599] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n12:178599] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n12:178599] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n06:138763] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n06:138763] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n06:138763] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n06:138763] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n06:138763] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n06:138763] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n06:138763] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n06:138763] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n06:161723] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n06:161723] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n06:161723] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n06:161723] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n06:161723] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n06:161723] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n06:161723] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n06:161723] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n06:161723] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n06:161723] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n10:16973] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n10:16973] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n10:16973] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n10:16973] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n10:16973] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n10:16973] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n10:16973] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n10:16973] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n17:176412] [ 6] [0x1000000504d8] +[b23n06:117528] [ 1] [0x3ffffb9dc980] +[b23n06:117528] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n06:117528] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n06:117528] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n11:03986] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n11:03986] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n11:03986] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n11:03986] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n11:03986] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n07:160800] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n07:160800] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n17:168155] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n17:168155] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n17:168155] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n17:168155] [ 8] [0x1000000504d8] +[a31n04:158632] [ 1] [0x3fffe8fd9430] +[a31n04:158632] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n04:158632] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n04:158632] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n04:158632] [ 5] [0x1000000504d8] +[b21n11:120584] [ 1] [0x3fffe434cc40] +[b21n11:120584] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n11:120584] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n11:120584] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n11:120584] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n14:178325] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n14:178325] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n14:178325] [ 9] [0x1000000504d8] +[b11n03:154293] [ 1] [0x3fffd730fc70] +[b11n03:154293] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n03:154293] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n09:119632] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n04:145748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n04:145748] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n18:179170] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n18:179170] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n18:179170] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n13:123514] [ 5] [0x1000000504d8] +[b02n13:171106] [ 1] [0x3ffff9d2ccb0] +[b02n13:171106] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n13:171106] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n05:139437] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n11:134517] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n11:134517] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n11:134517] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n11:134517] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n11:134517] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n11:134517] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n11:134517] [10] [0x3fffee60c470] +[b08n07:168961] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n07:168961] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n07:168961] [ 4] [b22n04:44759] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n17:169735] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n17:169735] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n17:169735] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n17:169735] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n17:169735] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n17:169735] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n10:11492] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n10:11492] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n10:11492] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n12:123562] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n12:123562] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n03:06823] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n03:06823] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n05:156008] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n05:156008] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n05:139913] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n05:139913] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n05:139913] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n05:139913] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n14:133966] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n14:133966] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n14:133966] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n14:133966] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n14:133966] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n14:133966] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n14:133966] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n14:133966] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n14:133966] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n14:133966] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n14:133966] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n01:163390] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n01:163390] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n01:163390] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n10:118390] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n10:118390] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n10:118390] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n10:118390] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n10:169210] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n10:169210] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n05:174780] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n05:174780] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n05:174780] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n05:174780] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n05:174780] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n13:123051] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n13:123051] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n13:154230] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n13:154230] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n13:154230] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n13:154230] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n13:154230] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n13:154230] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n13:154230] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n13:154230] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n04:03147] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n04:03147] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n04:03147] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n04:03147] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n04:03147] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n14:08399] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n14:08399] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n14:08399] [12] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n01:96749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n01:96749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n01:96749] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n01:96749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n01:96749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n01:96749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n01:96749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n12:160962] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n12:160962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n12:160962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n12:160962] [ 8] [0x1000000504d8] +[b09n08:176683] [ 1] [0x3ffffc66cd30] +[b09n08:176683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n08:176683] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n08:176683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n08:176683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n09:12245] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n09:12245] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n09:12245] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n09:12245] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n09:12245] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n11:130803] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n03:172650] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n03:172650] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n03:172650] [ 8] [b23n05:120228] [ 0] [0x1000000504d8] +[b23n05:120228] [ 1] [0x3ffff77d0750] +[b23n05:120228] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n05:120228] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n13:133627] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n01:170767] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n01:170767] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n01:170767] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n01:170767] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n01:170767] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n01:170767] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n01:170767] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n01:170767] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n16:154183] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n16:154183] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n16:154183] [13] [b18n15:120129] [ 0] [0x1000000504d8] +[b18n15:120129] [ 1] [0x3fffcc08ddc0] +[b18n15:120129] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n15:120129] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n10:140414] *** Process received signal *** +[a33n10:140414] Signal: Segmentation fault (11) +[a33n10:140414] Signal code: (3) +[a33n10:140414] Failing at address: 0x14170000000000a0 +[a33n10:140414] [ 0] [0x1000000504d8] +[a33n10:140414] [ 1] [0x3fffd97a2c00] +[a33n10:140414] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n10:140414] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n08:159025] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n08:159025] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n08:159025] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n08:159025] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n08:159025] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n08:159025] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n18:128544] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n18:128544] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n18:128544] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n18:128544] [10] [a33n11:156151] *** Process received signal *** +[a33n11:156151] Signal: Segmentation fault (11) +[a33n11:156151] Signal code: (3) +[a33n11:156151] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n02:122181] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n02:122181] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n02:122181] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n02:122181] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n14:152747] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n14:152747] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n07:165090] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n07:165090] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n07:165090] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n07:165090] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n07:165090] *** End of error message *** +[b13n15:139750] *** Process received signal *** +[b13n15:139750] Signal: Segmentation fault (11) +[b13n15:139750] Signal code: (3) +[b13n15:139750] Failing at address: 0xb170000000000a0 +[b13n15:139750] [ 0] [0x1000000504d8] +[b13n15:139750] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n05:04517] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n05:04517] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n05:04517] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n05:04517] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n05:04517] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n05:04517] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n05:04517] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n05:04517] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n05:04517] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n05:04517] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n15:122084] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n15:122084] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n15:122084] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n15:122084] [13] [b05n10:10925] *** Process received signal *** +[b05n10:10925] Signal: Segmentation fault (11) +[b05n10:10925] Signal code: (3) +[b05n10:10925] Failing at address: 0x14170000000000a0 +[b05n10:10925] [ 0] [0x1000000504d8] +[b05n10:10925] [ 1] [0x3fffce342770] +[b05n10:10925] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n10:10925] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n09:160373] *** End of error message *** +[b14n08:127442] *** Process received signal *** +[b14n08:127442] Signal: Segmentation fault (11) +[b14n08:127442] Signal code: (3) +[b14n08:127442] Failing at address: 0x14170000000000a0 +[b14n08:127442] [ 0] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n10:155400] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n10:155400] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n10:135850] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n10:135850] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n10:135850] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n10:135850] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n10:135850] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n10:135850] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n10:135850] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n10:135850] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n10:135850] *** End of error message *** +[0x1000000504d8] +[b09n10:169680] [ 1] [0x3fffff5a84f0] +[b09n10:169680] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n10:169680] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n10:169680] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n10:169680] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n02:74431] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n02:74431] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n02:74431] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n02:74431] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n02:74431] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n02:74431] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n02:74431] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n02:74431] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n02:74431] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n02:74431] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n07:172879] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n07:172879] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n03:164494] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n03:164494] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n03:164494] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n03:164494] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n16:124961] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n16:124961] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n16:124961] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n16:124961] [13] [0x1000000504d8] +[b19n01:121745] [ 1] [0x3fffcd079bf0] +[b19n01:121745] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n01:121745] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n01:121745] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n01:121745] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n01:121745] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n10:120977] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n10:120977] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n10:120977] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n10:120977] [ 9] [b19n04:130744] *** Process received signal *** +[b19n04:130744] Signal: Segmentation fault (11) +[b19n04:130744] Signal code: (3) +[b19n04:130744] Failing at address: 0xb170000000000a0 +[b19n04:130744] [ 0] [0x1000000504d8] +[b19n04:130744] [ 1] [0x3fffd64fc220] +[b19n04:130744] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n04:130744] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n04:130744] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n12:122875] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n12:122875] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n12:160190] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n12:160190] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n12:160190] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n12:160190] [ 9] [a31n13:155848] *** Process received signal *** +[a31n13:155848] Signal: Segmentation fault (11) +[a31n13:155848] Signal code: (3) +[a31n13:155848] Failing at address: 0xb170000000000a0 +[b04n07:02839] *** Process received signal *** +[b04n07:02839] Signal: Segmentation fault (11) +[b04n07:02839] Signal code: (3) +[b04n07:02839] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n07:45309] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n07:45309] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n07:45309] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n07:45309] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n07:45309] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n07:45309] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n07:45309] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n07:45309] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n07:45309] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n07:45309] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n07:45309] [14] [b07n05:176909] *** Process received signal *** +[b07n05:176909] Signal: Segmentation fault (11) +[b07n05:176909] Signal code: (3) +[b07n05:176909] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n09:170656] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n09:170656] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n10:122757] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n10:122757] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n08:146116] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n08:146116] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n08:146116] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n04:156414] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n04:156414] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n07:156489] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n03:135038] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n03:135038] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n03:135038] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n03:135038] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n03:135038] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n03:135038] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n09:128492] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n09:128492] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n09:128492] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n09:128492] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n10:176263] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n10:176263] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n10:176263] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n10:176263] *** End of error message *** +[b09n17:163177] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n14:126118] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n14:126118] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n14:126118] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n02:125373] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n02:125373] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n03:147533] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n03:147533] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n03:147533] [ 6] [b18n04:136993] *** Process received signal *** +[b18n04:136993] Signal: Segmentation fault (11) +[b18n04:136993] Signal code: (3) +[b18n04:136993] Failing at address: 0x14170000000000a0 +[b18n04:136993] [ 0] [0x1000000504d8] +[b18n04:136993] [ 1] [0x3fffc05ab9c0] +[b18n04:136993] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n04:136993] [ 3] [0x1000000504d8] +[b15n08:118551] [ 1] [0x3fffd5f36930] +[b15n08:118551] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n08:118551] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n08:118551] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n08:118551] [ 5] [b15n11:134746] *** Process received signal *** +[b15n11:134746] Signal: Segmentation fault (11) +[b15n11:134746] Signal code: (3) +[b15n11:134746] Failing at address: 0xb170000000000a0 +[b15n11:134746] [ 0] [0x1000000504d8] +[b15n11:134746] [ 1] [0x3fffcc7fd8a0] +[b15n11:134746] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n11:134746] [ 3] [0x1000000504d8] +[a26n03:68829] [ 1] [0x3fffc2a517d0] +[a26n03:68829] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n03:68829] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n03:68829] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n03:68829] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n03:68829] [ 6] [b09n05:173820] *** Process received signal *** +[b09n05:173820] Signal: Segmentation fault (11) +[b09n05:173820] Signal code: (3) +[b09n05:173820] Failing at address: 0xb170000000000a0 +[b09n05:173820] [ 0] [0x1000000504d8] +[b09n05:173820] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n14:122441] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n01:123569] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n01:123569] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n01:123569] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n12:145266] [ 4] [a32n08:165285] *** Process received signal *** +[a32n08:165285] Signal: Segmentation fault (11) +[a32n08:165285] Signal code: (3) +[a32n08:165285] Failing at address: 0x14170000000000a0 +[a32n08:165285] [ 0] [0x1000000504d8] +[a32n08:165285] [ 1] [0x3fffd8b20b00] +[a32n08:165285] [ 2] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n13:165387] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n13:165387] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n06:148311] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n06:148311] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n06:148311] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n06:148311] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n07:125450] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n13:137747] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n13:137747] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n13:137747] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n13:137747] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n03:119830] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n03:119830] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n03:119830] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n03:119830] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n03:119830] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n04:136608] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n08:124115] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n08:124115] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n08:124115] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n08:124115] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n03:124681] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n11:128954] *** End of error message *** +[b18n12:120404] Signal: Segmentation fault (11) +[b18n12:120404] Signal code: (3) +[b18n12:120404] Failing at address: 0x14170000000000a0 +[b18n12:120404] [ 0] [0x1000000504d8] +[b18n12:120404] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n07:163252] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n07:163252] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n07:163252] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n14:177221] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n14:177221] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n14:177221] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n14:177221] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n14:177221] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n14:177221] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n14:177221] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n14:177221] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n07:156139] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n07:156139] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n07:156139] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n05:05605] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n05:05605] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n05:05605] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n05:05605] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n18:121307] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n18:121307] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n18:121307] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n14:116411] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n16:129236] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n16:129236] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n16:129236] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n04:177598] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n04:177598] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n04:177598] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n04:177598] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n04:177598] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n09:143897] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n09:143897] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n14:139842] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n14:139842] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n14:139842] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n04:117352] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n08:144854] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n08:144854] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n08:144854] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n08:144854] [14] [a35n08:138177] *** Process received signal *** +[a35n08:138177] Signal: Segmentation fault (11) +[a35n08:138177] Signal code: (3) +[a35n08:138177] Failing at address: 0x5170000000000a0 +[a35n08:138177] [ 0] [0x1000000504d8] +[a35n08:138177] [ 1] [0x3ffffe401590] +[a35n08:138177] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n08:138177] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n08:138177] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n08:138177] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n08:138177] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n08:138177] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n08:138177] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n08:138177] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n08:138177] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n08:138177] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n08:138177] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n08:138177] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n08:138177] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n08:138177] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n12:177493] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n12:177493] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n12:177493] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n12:177493] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n12:177493] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n12:177493] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n05:53372] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n05:53372] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n05:53372] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n05:53372] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n05:53372] *** End of error message *** +[b02n09:166570] *** Process received signal *** +[b02n09:166570] Signal: Segmentation fault (11) +[b02n09:166570] Signal code: (3) +[b02n09:166570] Failing at address: 0x14170000000000a0 +[b02n09:166570] [ 0] [0x1000000504d8] +[b02n09:166570] [ 1] [0x3fffe5cf1d20] +[b02n09:166570] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n09:166570] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n09:166570] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n09:166570] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n09:166570] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n09:166570] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n09:166570] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n09:166570] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n09:166570] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n09:166570] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n09:166570] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n09:166570] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n09:166570] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n09:166570] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n12:180113] *** End of error message *** +[b19n18:122551] *** Process received signal *** +[b19n18:122551] Signal: Segmentation fault (11) +[b19n18:122551] Signal code: (3) +[b19n18:122551] Failing at address: 0xb170000000000a0 +[b19n18:122551] [ 0] [0x1000000504d8] +[b19n18:122551] [ 1] [0x3fffd11cd760] +[b19n18:122551] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n07:149475] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n07:149475] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n05:121821] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n05:121821] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n05:121821] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n05:121821] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n16:130376] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n16:130376] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n16:130376] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n16:130376] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n16:130376] [10] [a35n15:148164] *** Process received signal *** +[a35n15:148164] Signal: Segmentation fault (11) +[a35n15:148164] Signal code: (3) +[a35n15:148164] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n16:147286] *** End of error message *** +[0x1000000504d8] +[b10n10:126191] [ 1] [0x3ffff024fb30] +[b10n10:126191] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n10:126191] [ 3] [a35n13:138088] *** Process received signal *** +[a35n13:138088] Signal: Segmentation fault (11) +[a35n13:138088] Signal code: (3) +[a35n13:138088] Failing at address: 0xb170000000000a0 +[a35n13:138088] [ 0] [0x1000000504d8] +[a35n13:138088] [ 1] [0x3ffffe4ad460] +[a35n13:138088] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n13:138088] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n13:138088] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n13:138088] [ 5] [a27n15:32399] *** Process received signal *** +[a27n15:32399] Signal: Segmentation fault (11) +[a27n15:32399] Signal code: (3) +[a27n15:32399] Failing at address: 0x14170000000000a0 +[a27n15:32399] [ 0] [0x1000000504d8] +[a27n15:32399] [ 1] [0x3ffff56ac330] +[a27n15:32399] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n15:32399] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n18:03311] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n12:164383] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n12:164383] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n12:164383] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n04:168489] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n04:168489] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n04:168489] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n17:125805] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n14:136136] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n11:124446] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n01:163023] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n01:163023] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n15:119254] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n15:119254] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n15:120061] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n15:151248] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n15:151248] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n15:151248] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n11:163261] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n11:163261] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n11:163261] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n04:153963] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n04:153963] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n04:153963] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n12:178717] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n13:117507] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n13:117507] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n10:134734] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n10:134734] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n10:134734] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n10:134734] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n10:134734] [10] [0x3ffffc6ca580] +[a34n14:150130] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n14:150130] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n14:150130] [ 4] [0x3fffd0c810c0] +[b21n14:119486] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n14:119486] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n14:119486] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n15:124802] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n15:124802] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n15:124802] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n15:124802] [ 8] [b20n04:126347] *** Process received signal *** +[b20n04:126347] Signal: Segmentation fault (11) +[b20n04:126347] Signal code: (3) +[b20n04:126347] Failing at address: 0x14170000000000a0 +[b20n04:126347] [ 0] [0x1000000504d8] +[b20n04:126347] [ 1] [0x3ffff2e56380] +[b20n04:126347] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n04:126347] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n13:166924] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n13:166924] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n13:166924] [13] [b22n02:119531] *** Process received signal *** +[b22n02:119531] Signal: Segmentation fault (11) +[b22n02:119531] Signal code: (3) +[b22n02:119531] Failing at address: 0xb170000000000a0 +[b22n02:119531] [ 0] [0x1000000504d8] +[b22n02:119531] [ 1] [0x3ffff752a9d0] +[b22n02:119531] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n02:119531] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n11:162469] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n11:162469] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n11:162469] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n11:124432] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n11:124432] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n11:124432] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n13:168158] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n13:168158] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n03:158656] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n11:153415] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n11:153415] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n16:173550] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n05:125612] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n05:125612] *** End of error message *** +[b11n09:161229] *** Process received signal *** +[b11n09:161229] Signal: Segmentation fault (11) +[b11n09:161229] Signal code: (3) +[b11n09:161229] Failing at address: 0x14170000000000a0 +[b11n09:161229] [ 0] [0x1000000504d8] +[b11n09:161229] [ 1] [0x3fffdcaf8130] +[b11n09:161229] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n09:161229] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n18:140387] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n18:140387] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n18:140387] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n18:140387] [10] [a28n17:168162] *** Process received signal *** +[a28n17:168162] Signal: Segmentation fault (11) +[a28n17:168162] Signal code: (3) +[a28n17:168162] Failing at address: 0x14170000000000a0 +[b15n18:08430] *** Process received signal *** +[b15n18:08430] Signal: Segmentation fault (11) +[b15n18:08430] Signal code: (3) +[b15n18:08430] Failing at address: 0xb170000000000a0 +[b15n18:08430] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n04:166204] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n04:166204] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n16:128438] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n16:128438] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n16:128438] *** End of error message *** +[b19n13:123883] *** Process received signal *** +[b19n13:123883] Signal: Segmentation fault (11) +[b19n13:123883] Signal code: (3) +[b19n13:123883] Failing at address: 0xb170000000000a0 +[b19n13:123883] [ 0] [0x1000000504d8] +[b19n13:123883] [ 1] [0x3fffd880c470] +[b19n13:123883] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n13:123883] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n12:120128] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n12:120128] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n12:120128] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n12:120128] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n12:120128] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n12:120128] [10] [b13n17:129930] *** Process received signal *** +[b13n17:129930] Signal: Segmentation fault (11) +[b13n17:129930] Signal code: (3) +[b13n17:129930] Failing at address: 0xb170000000000a0 +[b13n17:129930] [ 0] [0x1000000504d8] +[b13n17:129930] [ 1] [0x3fffdd76d5e0] +[b13n17:129930] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n17:129930] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n08:116174] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n08:116174] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n08:116174] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n08:54627] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n18:131584] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n17:171198] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n17:171198] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n14:133064] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n14:133064] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n14:133064] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n14:133064] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n14:133064] [ 8] [a32n10:162607] [ 0] [0x1000000504d8] +[a32n10:162607] [ 1] [0x3fffe0e12330] +[a32n10:162607] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n10:162607] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n10:162607] [ 4] [a33n02:167598] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n02:167598] [13] [b09n18:164800] *** Process received signal *** +[b09n18:164800] Signal: Segmentation fault (11) +[b09n18:164800] Signal code: (3) +[b09n18:164800] Failing at address: 0xb170000000000a0 +[b09n18:164800] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n08:131379] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n08:131379] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n08:131379] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n06:127275] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n06:127275] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n06:127275] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n06:127275] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n17:124163] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n16:165925] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n16:165925] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n05:164945] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n05:164945] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n05:164945] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n05:164945] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n05:164945] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n05:164945] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n05:164945] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n05:164945] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n05:164945] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n07:117358] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n07:117358] [ 5] [b13n07:133376] *** Process received signal *** +[b13n07:133376] Signal: Segmentation fault (11) +[b13n07:133376] Signal code: (3) +[b13n07:133376] Failing at address: 0xb170000000000a0 +[b13n07:133376] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n12:132099] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n12:132099] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n12:132099] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n04:41671] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n04:41671] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n04:41671] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n04:41671] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n04:41671] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n04:41671] *** End of error message *** +[b05n02:176047] *** Process received signal *** +[b05n02:176047] Signal: Segmentation fault (11) +[b05n02:176047] Signal code: (3) +[b05n02:176047] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n18:07421] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n18:07421] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n18:07421] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n18:07421] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n18:07421] *** End of error message *** +[b17n16:127350] *** Process received signal *** +[b17n16:127350] Signal: Segmentation fault (11) +[b17n16:127350] Signal code: (3) +[b17n16:127350] Failing at address: 0xb170000000000a0 +[a31n06:167371] *** Process received signal *** +[a31n06:167371] Signal: Segmentation fault (11) +[a31n06:167371] Signal code: (3) +[a31n06:167371] Failing at address: 0x14170000000000a0 +[a31n06:167371] [ 0] [0x1000000504d8] +[a31n06:167371] [ 1] [0x3fffcfc0e1d0] +[a31n06:167371] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n03:38502] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n14:176069] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n14:176069] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n03:123615] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n03:123615] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n03:123615] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n03:123615] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n03:123615] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n11:170895] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n11:170895] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n11:170895] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n11:170895] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n11:170895] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n11:170895] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n01:128830] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n01:128830] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n01:128830] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n01:128830] [ 8] [b19n11:133972] *** Process received signal *** +[b19n11:133972] Signal: Segmentation fault (11) +[b19n11:133972] Signal code: (3) +[b19n11:133972] Failing at address: 0x14170000000000a0 +[b19n11:133972] [ 0] [0x1000000504d8] +[b19n11:133972] [ 1] [0x3fffc89e15e0] +[b19n11:133972] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n11:133972] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n11:133972] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n11:133972] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n11:133972] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n11:133972] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n11:133972] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n11:133972] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n11:133972] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n11:133972] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n11:133972] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n11:133972] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n11:133972] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n11:133972] *** End of error message *** +[b22n13:122029] [ 0] [0x1000000504d8] +[b22n13:122029] [ 1] [0x3fffcd19ef90] +[b22n13:122029] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n13:122029] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n13:122029] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n07:180193] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n16:157885] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n16:157885] [ 9] [b10n11:160349] *** Process received signal *** +[b10n11:160349] Signal: Segmentation fault (11) +[b10n11:160349] Signal code: (3) +[b10n11:160349] Failing at address: 0x14170000000000a0 +[b10n11:160349] [ 0] [0x1000000504d8] +[b10n11:160349] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n10:142932] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n10:142932] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n18:179697] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n18:179697] [ 5] [b23n13:114394] *** Process received signal *** +[b23n13:114394] Signal: Segmentation fault (11) +[b23n13:114394] Signal code: (3) +[b23n13:114394] Failing at address: 0xb170000000000a0 +[b23n13:114394] [ 0] [0x1000000504d8] +[b23n13:114394] [ 1] [0x3fffdcfe86b0] +[b23n13:114394] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n13:114394] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n13:114394] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n13:114394] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n13:114394] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n16:116522] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n16:116522] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n12:12764] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n12:12764] *** End of error message *** +[b03n12:167675] *** Process received signal *** +[b03n12:167675] Signal: Segmentation fault (11) +[b03n12:167675] Signal code: (3) +[b03n12:167675] Failing at address: 0xb170000000000a0 +[b03n12:167675] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n09:51334] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n09:51334] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n09:51334] [12] [a34n17:149906] *** Process received signal *** +[a34n17:149906] Signal: Segmentation fault (11) +[a34n17:149906] Signal code: (3) +[a34n17:149906] Failing at address: 0xb170000000000a0 +[a34n17:149906] [ 0] [0x1000000504d8] +[a34n17:149906] [ 1] [0x3fffdb44bbe0] +[a34n17:149906] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n17:149906] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n09:123418] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n15:177054] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n15:177054] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n06:126379] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n07:118163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n07:118163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n07:118163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n07:118163] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n07:118163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n07:118163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n07:118163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n07:118163] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n15:155429] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n15:155429] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n15:155429] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n15:155429] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n15:155429] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n15:155429] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n15:155429] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n15:155429] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n15:155429] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n15:155429] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n02:06563] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n02:06563] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n02:06563] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n02:06563] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n02:06563] [10] [0x3fffe7e1c330] +[a27n01:48803] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n01:48803] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n01:48803] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n01:48803] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n14:128867] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n14:128867] [ 4] [0x1000000504d8] +[b13n05:135163] [ 1] [0x3fffdf4a94a0] +[b13n05:135163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n05:135163] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n05:135163] [ 4] [b22n12:122967] *** Process received signal *** +[b22n12:122967] Signal: Segmentation fault (11) +[b22n12:122967] Signal code: (3) +[b22n12:122967] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n14:133635] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n14:133635] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n14:133635] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n17:15870] [ 0] [0x1000000504d8] +[a27n17:15870] [ 1] [0x3fffebe86950] +[a27n17:15870] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n17:15870] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n17:15870] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n17:15870] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n17:15870] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n14:169520] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n14:169520] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n01:167298] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n14:136659] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n14:136659] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n14:136659] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n14:136659] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n14:136659] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n14:136659] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n14:136659] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n14:136659] [12] [a28n13:176363] *** Process received signal *** +[a28n13:176363] Signal: Segmentation fault (11) +[a28n13:176363] Signal code: (3) +[a28n13:176363] Failing at address: 0xb170000000000a0 +[a28n13:176363] [ 0] [0x1000000504d8] +[a28n13:176363] [ 1] [0x3fffdf31a660] +[a28n13:176363] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n13:176363] [ 3] [a33n01:141814] *** Process received signal *** +[a33n01:141814] Signal: Segmentation fault (11) +[a33n01:141814] Signal code: (3) +[a33n01:141814] Failing at address: 0xb170000000000a0 +[a33n01:141814] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n06:01840] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n06:01840] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n06:01840] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n06:01840] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n10:176310] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n10:176310] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n10:176310] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n10:176310] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n08:08956] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n08:08956] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n08:08956] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n08:08956] *** End of error message *** +[0x1000000504d8] +[a35n07:138204] [ 1] [0x3fffce036640] +[a35n07:138204] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n07:138204] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n18:126903] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n06:143008] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n06:143008] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n06:143008] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n07:132086] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n07:132086] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n04:169913] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n04:169913] *** End of error message *** +[a33n08:147819] *** Process received signal *** +[a33n08:147819] Signal: Segmentation fault (11) +[a33n08:147819] Signal code: (3) +[a33n08:147819] Failing at address: 0xb170000000000a0 +[a33n08:147819] [ 0] [0x1000000504d8] +[a33n08:147819] [ 1] [b10n02:164491] *** Process received signal *** +[b10n02:164491] Signal: Segmentation fault (11) +[b10n02:164491] Signal code: (3) +[b10n02:164491] Failing at address: 0xb170000000000a0 +[b10n02:164491] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n07:153175] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n07:153175] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n07:153175] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n07:153175] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n07:153175] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n07:153175] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n07:153175] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n02:166746] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n02:166746] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n07:131673] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n07:131673] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n07:131673] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n07:131673] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n07:131673] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n07:131673] *** End of error message *** +[a36n17:150931] *** Process received signal *** +[a36n17:150931] Signal: Segmentation fault (11) +[a36n17:150931] Signal code: (3) +[a36n17:150931] Failing at address: 0x14170000000000a0 +[b12n11:137870] *** Process received signal *** +[b12n11:137870] Signal: Segmentation fault (11) +[b12n11:137870] Signal code: (3) +[b12n11:137870] Failing at address: 0xb170000000000a0 +[b12n11:137870] [ 0] [0x1000000504d8] +[b12n11:137870] [ 1] [0x3fffe48e4710] +[b12n11:137870] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n11:137870] [ 3] [b02n16:01568] *** Process received signal *** +[b02n16:01568] Signal: Segmentation fault (11) +[b02n16:01568] Signal code: (3) +[b02n16:01568] Failing at address: 0xb170000000000a0 +[b02n16:01568] [ 0] [0x1000000504d8] +[b02n16:01568] [ 1] [0x3ffff3345b20] +[b02n16:01568] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n16:01568] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n01:128154] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n01:128154] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n03:10289] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n03:10289] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n09:121752] [ 4] [b15n13:123705] *** Process received signal *** +[b15n13:123705] Signal: Segmentation fault (11) +[b15n13:123705] Signal code: (3) +[b15n13:123705] Failing at address: 0x14170000000000a0 +[b15n13:123705] [ 0] [0x1000000504d8] +[b15n13:123705] [ 1] [b11n11:152985] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n11:171964] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n11:171964] *** End of error message *** +[a27n18:28219] *** Process received signal *** +[a27n18:28219] Signal: Segmentation fault (11) +[a27n18:28219] Signal code: (3) +[a27n18:28219] Failing at address: 0xb170000000000a0 +[a27n18:28219] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n11:01172] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n11:01172] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n11:01172] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n11:01172] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n11:01172] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n08:174408] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n08:174408] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n08:174408] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n08:174408] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n08:174408] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n08:174408] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n08:124723] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n08:124723] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n08:124723] [ 7] [0x1000000504d8] +[b01n09:173753] [ 1] [0x3fffc4336da0] +[b01n09:173753] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n09:173753] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n09:173753] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n15:85942] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n15:85942] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n15:85942] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n01:130409] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n01:130409] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n01:130409] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n01:130409] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n01:130409] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n01:130409] [11] [b11n14:161619] *** Process received signal *** +[b11n14:161619] Signal: Segmentation fault (11) +[b11n14:161619] Signal code: (3) +[b11n14:161619] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n10:130579] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n10:130579] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n10:130579] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n10:130579] [ 9] [b12n02:143965] *** Process received signal *** +[b12n02:143965] Signal: Segmentation fault (11) +[b12n02:143965] Signal code: (3) +[b12n02:143965] Failing at address: 0x5170000000000a0 +[b12n02:143965] [ 0] [b14n16:122492] *** Process received signal *** +[b14n16:122492] Signal: Segmentation fault (11) +[b14n16:122492] Signal code: (3) +[b14n16:122492] Failing at address: 0xb170000000000a0 +[b14n16:122492] [ 0] [b06n04:175996] [ 0] [0x1000000504d8] +[b06n04:175996] [ 1] [0x3ffffa5a2770] +[b06n04:175996] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n04:175996] [ 3] [b21n06:122163] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n12:148382] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n12:11146] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n12:11146] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n12:11146] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n12:11146] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n12:11146] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n12:11146] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n12:11146] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n12:11146] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n12:11146] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n12:11146] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n09:119545] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n09:119545] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n09:119545] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n01:173781] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n01:173781] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n01:173781] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n01:173781] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n01:173781] [10] [a31n12:156914] *** Process received signal *** +[a31n12:156914] Signal: Segmentation fault (11) +[a31n12:156914] Signal code: (3) +[a31n12:156914] Failing at address: 0xb170000000000a0 +[a31n12:156914] [ 0] [0x1000000504d8] +[a31n12:156914] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n03:127628] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n03:127628] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n03:127628] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n03:127628] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n09:123811] *** End of error message *** +[b21n02:120189] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n16:171915] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n16:171915] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n16:171915] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n16:171915] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n16:171915] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n16:171915] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n16:171915] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n16:171915] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n16:171915] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n16:171915] *** End of error message *** +[b22n17:119579] *** Process received signal *** +[b22n17:119579] Signal: Segmentation fault (11) +[b22n17:119579] Signal code: (3) +[b22n17:119579] Failing at address: 0x14170000000000a0 +[b22n17:119579] [ 0] [0x1000000504d8] +[b22n17:119579] [ 1] [0x3fffe5675cc0] +[b22n17:119579] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n17:119579] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n16:153291] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n16:153291] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n16:153291] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n16:153291] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n15:148583] *** Process received signal *** +[a33n15:148583] Signal: Segmentation fault (11) +[a33n15:148583] Signal code: (3) +[a33n15:148583] Failing at address: 0x5170000000000a0 +[a33n15:148583] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n01:145734] *** End of error message *** +[b12n04:138156] *** Process received signal *** +[b12n04:138156] Signal: Segmentation fault (11) +[b12n04:138156] Signal code: (3) +[b12n04:138156] Failing at address: 0x5170000000000a0 +[b12n04:138156] [ 0] [b06n02:163318] [ 0] [0x1000000504d8] +[b06n02:163318] [ 1] [0x3fffea052c40] +[b06n02:163318] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n02:163318] [ 3] [0x1000000504d8] +[b18n10:122206] [ 1] [0x3fffcc3caac0] +[b18n10:122206] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n10:122206] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n10:122206] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n10:122206] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n10:122206] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n10:122206] [ 7] [b17n07:125471] *** Process received signal *** +[b17n07:125471] Signal: Segmentation fault (11) +[b17n07:125471] Signal code: (3) +[b17n07:125471] Failing at address: 0xb170000000000a0 +[b17n07:125471] [ 0] [0x1000000504d8] +[b17n07:125471] [ 1] [0x3fffe16a98d0] +[b17n07:125471] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n07:125471] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n07:125471] [ 4] [0x1000000504d8] +[a34n08:163163] [ 1] [0x3fffdfe19440] +[a34n08:163163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n08:163163] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n08:163163] [ 4] [b10n17:164563] *** Process received signal *** +[b10n17:164563] Signal: Segmentation fault (11) +[b10n17:164563] Signal code: (3) +[b10n17:164563] Failing at address: 0xb170000000000a0 +[b10n17:164563] [ 0] [0x1000000504d8] +[b10n17:164563] [ 1] [0x3fffc71f2280] +[b10n17:164563] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n17:164563] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n18:163109] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n18:163109] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n18:163109] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n18:163109] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n18:163109] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n18:163109] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n18:163109] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n18:163109] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n18:163109] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n18:163109] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n18:163109] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n18:163109] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n11:147713] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n03:168208] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n03:168208] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n03:168208] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n03:168208] [14] [b06n14:173892] [ 0] [0x1000000504d8] +[b06n14:173892] [ 1] [0x3ffff0173140] +[b06n14:173892] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n14:173892] [ 3] [b14n07:127358] *** Process received signal *** +[b14n07:127358] Signal: Segmentation fault (11) +[b14n07:127358] Signal code: (3) +[b14n07:127358] Failing at address: 0xb170000000000a0 +[b14n07:127358] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n01:165162] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n01:165162] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n01:165162] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n01:165162] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n01:165162] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n01:165162] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n01:165162] [13] [a31n08:156369] *** Process received signal *** +[a31n08:156369] Signal: Segmentation fault (11) +[a31n08:156369] Signal code: (3) +[a31n08:156369] Failing at address: 0xb170000000000a0 +[a31n08:156369] [ 0] [0x1000000504d8] +[a31n08:156369] [ 1] [0x3fffe77bc660] +[a31n08:156369] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n08:156369] [ 3] [0x3fffe3a6e0b0] +[b23n12:115640] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n12:115640] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n12:115640] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n12:115640] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n13:163195] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n04:169410] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n04:169410] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n04:169410] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n04:169410] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n04:169410] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n04:169410] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n04:169410] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n04:169410] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n04:169410] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n13:165262] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n13:165262] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n13:165262] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n13:165262] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n12:177120] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n12:177120] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n12:177120] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n02:155382] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n08:163100] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n08:163100] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n05:169785] [ 9] [0x1000000504d8] +[b19n08:121686] [ 1] [0x3ffffafab100] +[b19n08:121686] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n08:121686] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n08:121686] [ 4] [ 0] [0x1000000504d8] +[b17n15:137005] [ 1] [0x3fffcf617340] +[b17n15:137005] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n15:137005] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n15:137005] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n15:154383] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n15:154383] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n15:154383] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n15:154383] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n15:154383] [14] [b22n11:120239] *** Process received signal *** +[b22n11:120239] Signal: Segmentation fault (11) +[b22n11:120239] Signal code: (3) +[b22n11:120239] Failing at address: 0xb170000000000a0 +[b22n11:120239] [ 0] [0x1000000504d8] +[b22n11:120239] [ 1] [0x3fffdf356980] +[b22n11:120239] [ 2] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n02:149226] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n02:149226] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n11:175597] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n11:175597] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n11:175597] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n11:175597] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n11:175597] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n11:175597] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n11:175597] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n11:175597] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n11:175597] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n11:175597] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n11:175597] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n11:175597] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n11:152816] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n11:152816] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n11:152816] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n01:138004] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n11:02683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n17:146793] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n17:146793] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n17:146793] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n17:146793] [10] [b13n12:140123] *** Process received signal *** +[b13n12:140123] Signal: Segmentation fault (11) +[b13n12:140123] Signal code: (3) +[b13n12:140123] Failing at address: 0xb170000000000a0 +[b13n12:140123] [ 0] [0x1000000504d8] +[b13n12:140123] [ 1] [0x3fffe4d8c670] +[b13n12:140123] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n12:140123] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n12:140123] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n12:140123] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n12:140123] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n12:140123] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n12:140123] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n12:140123] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n12:140123] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n12:140123] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n12:140123] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n12:140123] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n12:140123] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n12:140123] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n17:126524] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n06:10225] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n06:10225] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n06:10225] *** End of error message *** +[b10n03:168933] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n09:164364] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n09:164364] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n15:123037] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n15:123037] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n14:163549] [ 7] [a35n03:135162] Signal: Segmentation fault (11) +[a35n03:135162] Signal code: (3) +[a35n03:135162] Failing at address: 0xb170000000000a0 +[b11n12:154344] *** Process received signal *** +[b11n12:154344] Signal: Segmentation fault (11) +[b11n12:154344] Signal code: (3) +[b11n12:154344] Failing at address: 0xb170000000000a0 +[b11n12:154344] [ 0] [0x1000000504d8] +[b11n12:154344] [ 1] [0x3fffe4a736d0] +[b11n12:154344] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n12:154344] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n12:154344] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n16:124794] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n16:124794] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n16:124794] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n16:124794] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n16:124794] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n16:124794] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n16:124794] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n16:124794] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n16:124794] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n16:124794] *** End of error message *** +[ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n12:11419] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n12:11419] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n12:11419] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n15:161756] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n15:161756] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n15:161756] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n15:161756] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n06:177003] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n01:65728] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n01:65728] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n17:119464] [ 4] [b23n03:125242] [ 0] [0x1000000504d8] +[b23n03:125242] [ 1] [0x3fffec703600] +[b23n03:125242] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n03:125242] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n03:125242] [ 4] [a32n06:154620] *** Process received signal *** +[a32n06:154620] Signal: Segmentation fault (11) +[a32n06:154620] Signal code: (3) +[a32n06:154620] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n04:169704] *** End of error message *** +[b18n03:136664] *** Process received signal *** +[b18n03:136664] Signal: Segmentation fault (11) +[b18n03:136664] Signal code: (3) +[b18n03:136664] Failing at address: 0x14170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n05:162566] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n11:148674] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n02:178474] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n02:178474] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n02:178474] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n02:178474] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n03:01353] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n03:01353] [ 6] [0x1000000504d8] +[a33n09:147793] [ 1] [0x3fffcc3dfec0] +[a33n09:147793] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n09:147793] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n09:147793] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n02:122556] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n02:122556] *** End of error message *** +[10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n10:161716] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n10:161716] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n10:161716] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n02:127218] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n02:123702] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n02:123702] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n17:162357] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n17:162357] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n06:133955] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n01:164964] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n01:164964] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n01:164964] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n01:164964] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n01:164964] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n01:164964] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n01:164964] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n01:164964] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n17:163819] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n17:163819] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n17:163819] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n17:163819] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n17:163819] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n17:163819] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n17:163819] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n17:163819] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n17:163819] [12] [b18n05:135120] [ 0] [0x1000000504d8] +[b18n05:135120] [ 1] [0x3fffc11cd870] +[b18n05:135120] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n05:135120] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n05:135120] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n06:130540] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n04:13079] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n04:13079] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n04:13079] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n04:13079] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n04:13079] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n04:13079] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n04:13079] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n04:13079] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n03:151879] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n03:151879] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n03:151879] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n03:151879] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n05:131263] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n05:131263] [ 5] [a28n01:179259] *** Process received signal *** +[a28n01:179259] Signal: Segmentation fault (11) +[a28n01:179259] Signal code: (3) +[a28n01:179259] Failing at address: 0xb170000000000a0 +[a36n02:147881] *** Process received signal *** +[a36n02:147881] Signal: Segmentation fault (11) +[a36n02:147881] Signal code: (3) +[a36n02:147881] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n01:171786] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n01:171786] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n01:171786] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n01:171786] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n05:154188] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n05:154188] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n05:154188] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n05:154188] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n05:154188] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n05:154188] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n05:154188] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n05:154188] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n17:118575] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n17:118575] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n17:118575] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n17:118575] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n17:118575] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n04:142255] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n04:142255] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n04:142255] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n04:142255] [13] [a36n01:141343] *** Process received signal *** +[a36n01:141343] Signal: Segmentation fault (11) +[a36n01:141343] Signal code: (3) +[a36n01:141343] Failing at address: 0x5170000000000a0 +[a36n01:141343] [ 0] [0x1000000504d8] +[a36n01:141343] [ 1] [0x3fffcbee33e0] +[a36n01:141343] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n01:141343] [ 3] [0x3fffd8cc1580] +[b01n08:172431] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n08:172431] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n08:172431] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n08:172431] [ 5] [a36n03:153850] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n18:118681] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n18:118681] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n18:118681] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n16:121930] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n16:121930] [ 6] [0x1000000504d8] +[a29n10:164936] [ 1] [0x3fffd01a75e0] +[a29n10:164936] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n10:164936] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n10:164936] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n10:164936] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n10:164936] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n10:164936] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n10:164936] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n10:164936] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n10:164936] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n10:164936] [11] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n08:164749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n08:164749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n11:164993] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n11:164993] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n11:164993] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n11:164993] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n14:152037] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n14:152037] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n14:152037] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n06:162915] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n04:04848] [ 5] [b01n06:171501] *** Process received signal *** +[b01n06:171501] Signal: Segmentation fault (11) +[b01n06:171501] Signal code: (3) +[b01n06:171501] Failing at address: 0xb170000000000a0 +[b01n06:171501] [ 0] [0x1000000504d8] +[b01n06:171501] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n03:130349] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n03:130349] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n03:130349] [ 8] [0x3fffe6e86810] +[b15n06:120732] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n06:120732] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n06:120732] [ 4] [b10n05:173079] *** Process received signal *** +[b10n05:173079] Signal: Segmentation fault (11) +[b10n05:173079] Signal code: (3) +[b10n05:173079] Failing at address: 0xb170000000000a0 +[b10n05:173079] [ 0] [0x1000000504d8] +[b10n05:173079] [ 1] [0x3fffe73dded0] +[b10n05:173079] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n05:173079] [ 3] [b14n02:139026] *** Process received signal *** +[b14n02:139026] Signal: Segmentation fault (11) +[b14n02:139026] Signal code: (3) +[b14n02:139026] Failing at address: 0x14170000000000a0 +[b14n02:139026] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n06:163674] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n06:163674] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n06:163674] [ 6] [b14n01:128002] *** Process received signal *** +[b14n01:128002] Signal: Segmentation fault (11) +[b14n01:128002] Signal code: (3) +[b14n01:128002] Failing at address: 0xb170000000000a0 +[b12n12:138734] Signal: Segmentation fault (11) +[b12n12:138734] Signal code: (3) +[b12n12:138734] Failing at address: 0x14170000000000a0 +[b12n12:138734] [ 0] [0x1000000504d8] +[b12n12:138734] [ 1] [0x3fffd7e2e7b0] +[b12n12:138734] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n12:138734] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n12:138734] [ 4] [0x1000000504d8] +[b15n05:120329] [ 1] [0x3fffeddfe0f0] +[b15n05:120329] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n05:120329] [ 3] [0x1000000504d8] +[b06n11:160795] [ 1] [0x3fffe18f1180] +[b06n11:160795] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n11:160795] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n11:160795] [ 4] [b19n17:123237] *** Process received signal *** +[b19n17:123237] Signal: Segmentation fault (11) +[b19n17:123237] Signal code: (3) +[b19n17:123237] Failing at address: 0x14170000000000a0 +[b19n17:123237] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n16:164550] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n04:126858] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n04:126858] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n04:126858] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n05:126778] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n17:176412] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n17:176412] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n17:176412] [ 9] [0x1000000504d8] +[b10n01:165123] [ 1] [0x3fffda6bb5e0] +[b10n01:165123] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n01:165123] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n01:165123] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n07:124348] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n07:124348] [ 6] [0x1000000504d8] +[a32n09:155759] [ 1] [0x3fffdb25a2a0] +[a32n09:155759] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n09:155759] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n09:155759] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n13:10195] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n13:10195] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n13:10195] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n13:10195] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n13:153676] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n13:153676] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n13:153676] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n13:153676] [13] [a29n06:157788] *** Process received signal *** +[a29n06:157788] Signal: Segmentation fault (11) +[a29n06:157788] Signal code: (3) +[a29n06:157788] Failing at address: 0xb170000000000a0 +[a29n06:157788] [ 0] [0x1000000504d8] +[a29n06:157788] [ 1] [0x3fffff3c6000] +[a29n06:157788] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n06:157788] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n11:117473] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n11:117473] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n11:117473] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n12:178600] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n12:178600] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n12:178600] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n12:178600] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n06:138762] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n06:138762] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n06:161724] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n06:161724] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n06:161724] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n06:161724] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n06:161724] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n11:03984] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n11:03984] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n10:16974] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n10:16974] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n10:16974] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n10:16974] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n10:16974] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n10:16974] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n10:16974] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n06:117528] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n06:117528] [ 6] [0x1000000504d8] +[b21n11:120583] [ 1] [0x3fffd7141760] +[b21n11:120583] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n11:120583] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n11:120583] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n11:120583] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n11:120583] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n07:160801] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n07:160801] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n14:178326] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n14:178326] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n14:178326] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n14:178326] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n14:178326] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n04:158631] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n04:158631] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n04:158631] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n17:168154] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n17:168154] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n17:168154] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n17:168154] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n03:154293] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n18:179169] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n18:179169] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n18:179169] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n18:179169] [14] [b23n09:119631] *** Process received signal *** +[b23n09:119631] Signal: Segmentation fault (11) +[b23n09:119631] Signal code: (3) +[b23n09:119631] Failing at address: 0xb170000000000a0 +[b23n09:119631] [ 0] [0x1000000504d8] +[b23n09:119631] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n04:145748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n04:145748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n04:145748] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n13:123514] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n13:171106] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n05:139436] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n05:139436] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n05:139436] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n05:139436] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n05:139436] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n05:139436] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n05:139436] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n05:139436] *** End of error message *** +[0x1000000504d8] +[b22n04:44758] [ 1] [0x3fffc6ce5d80] +[b22n04:44758] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n04:44758] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n04:44758] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n11:134518] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n11:134518] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n11:134518] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n11:134518] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n11:134518] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n11:134518] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n11:134518] *** End of error message *** +[0x3fffdcd34c40] +[b08n07:168960] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n07:168960] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n07:168960] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n07:168960] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n10:11491] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n10:11491] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n10:11491] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n10:11491] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n12:123561] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n12:123561] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n03:06823] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n03:06823] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n03:06823] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n05:139912] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n05:139912] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n05:139912] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n05:139912] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n14:133970] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n01:163390] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n10:118390] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n05:174780] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n10:169209] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n10:169209] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n10:169209] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n10:169209] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n10:169209] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n10:169209] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n10:169209] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n10:169209] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n10:169209] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n13:123051] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n13:154231] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n13:154231] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n14:08399] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n14:08399] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n04:03148] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n04:03148] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n04:03148] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n04:03148] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n01:96749] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n01:96749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n01:96749] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n01:96749] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n12:160963] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n12:160963] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n12:160963] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n09:12246] *** End of error message *** +[b09n08:176682] [ 1] [0x3ffff6cf72e0] +[b09n08:176682] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n08:176682] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n08:176682] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n08:176682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n08:176682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n08:176682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n08:159024] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n08:159024] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n08:159024] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n08:159024] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n08:159024] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n08:159024] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n05:120227] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n03:172649] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n03:172649] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n03:172649] [ 9] [b13n13:133626] *** Process received signal *** +[b13n13:133626] Signal: Segmentation fault (11) +[b13n13:133626] Signal code: (3) +[b13n13:133626] Failing at address: 0x14170000000000a0 +[b13n13:133626] [ 0] [0x1000000504d8] +[b13n13:133626] [ 1] [0x3fffce8d6090] +[b13n13:133626] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n13:133626] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n01:170766] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n01:170766] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n01:170766] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n01:170766] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n01:170766] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n01:170766] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n16:154183] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n16:154183] *** End of error message *** +[0x1000000504d8] +[b18n15:120130] [ 1] [0x3fffd20ed1e0] +[b18n15:120130] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n15:120130] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n15:120130] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n15:120130] [ 5] [0x1000000504d8] +[a33n10:140413] [ 1] [0x3fffe1d02c00] +[a33n10:140413] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n10:140413] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n10:140413] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n10:140413] [ 5] [a28n06:179454] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n18:128543] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n18:128543] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n18:128543] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n18:128543] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n18:128543] [10] [a33n11:156152] *** Process received signal *** +[a33n11:156152] Signal: Segmentation fault (11) +[a33n11:156152] Signal code: (3) +[a33n11:156152] Failing at address: 0x5170000000000a0 +[a33n11:156152] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n02:122181] *** End of error message *** +[a32n14:152745] *** Process received signal *** +[a32n14:152745] Signal: Segmentation fault (11) +[a32n14:152745] Signal code: (3) +[a32n14:152745] Failing at address: 0xb170000000000a0 +[a32n14:152745] [ 0] [0x1000000504d8] +[a32n14:152745] [ 1] [0x3ffffc5f3d30] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n11:130803] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n11:130803] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n11:130803] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n11:130803] [ 9] [0x3fffc8563050] +[b13n15:139749] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n15:139749] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n15:139749] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n15:139749] [ 5] [a34n07:165087] *** Process received signal *** +[a34n07:165087] Signal: Segmentation fault (11) +[a34n07:165087] Signal code: (3) +[a34n07:165087] Failing at address: 0xb170000000000a0 +[a34n07:165087] [ 0] [0x1000000504d8] +[a34n07:165087] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n10:10926] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n15:122085] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n15:122085] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n15:122085] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n15:122085] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n15:122085] *** End of error message *** +[a02n09:160370] *** Process received signal *** +[a02n09:160370] Signal: Segmentation fault (11) +[a02n09:160370] Signal code: (3) +[a02n09:160370] Failing at address: 0xb170000000000a0 +[a02n09:160370] [ 0] [0x1000000504d8] +[a02n09:160370] [ 1] [0x3ffffd4aa570] +[a02n09:160370] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n09:160370] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n09:160370] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n09:160370] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n09:160370] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n09:160370] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n09:160370] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n09:160370] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n09:160370] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n09:160370] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n09:160370] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n09:160370] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n09:160370] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n09:160370] *** End of error message *** +[b14n08:127443] *** Process received signal *** +[b14n08:127443] Signal: Segmentation fault (11) +[b14n08:127443] Signal code: (3) +[b14n08:127443] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n10:155399] *** Process received signal *** +[a31n10:155399] Signal: Segmentation fault (11) +[a31n10:155399] Signal code: (3) +[a31n10:155399] Failing at address: 0x14170000000000a0 +[a31n10:155399] [ 0] [0x1000000504d8] +[a31n10:155399] [ 1] [0x3ffff66ebae0] +[a31n10:155399] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n10:155399] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n10:169680] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n10:169680] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n10:169680] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n10:135851] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n10:135851] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n10:135851] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n10:135851] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n10:135851] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n10:135851] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n02:74430] [ 9] [a32n07:172880] *** Process received signal *** +[a32n07:172880] Signal: Segmentation fault (11) +[a32n07:172880] Signal code: (3) +[a32n07:172880] Failing at address: 0x14170000000000a0 +[a32n07:172880] [ 0] [0x1000000504d8] +[a32n07:172880] [ 1] [0x3ffff69a3ad0] +[a32n07:172880] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n07:172880] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n16:124960] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n16:124960] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n16:124960] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n01:121745] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n01:121745] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n01:121745] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n03:164494] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n03:164494] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n10:120978] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n10:120978] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n10:120978] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n10:120978] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n10:120978] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n12:160191] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n12:160191] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n12:160191] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n12:160191] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n12:160191] [10] [a31n13:155847] *** Process received signal *** +[a31n13:155847] Signal: Segmentation fault (11) +[a31n13:155847] Signal code: (3) +[a31n13:155847] Failing at address: 0xb170000000000a0 +[a31n13:155847] [ 0] [0x1000000504d8] +[a31n13:155847] [ 1] [0x3fffdcba4d70] +[a31n13:155847] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n13:155847] [ 3] [0x1000000504d8] +[b19n04:130745] [ 1] [0x3fffd22561d0] +[b19n04:130745] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n04:130745] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n04:130745] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n04:130745] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n04:130745] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n04:130745] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n04:130745] [ 8] [b19n12:122873] *** Process received signal *** +[b19n12:122873] Signal: Segmentation fault (11) +[b19n12:122873] Signal code: (3) +[b19n12:122873] Failing at address: 0x14170000000000a0 +[b19n12:122873] [ 0] [0x1000000504d8] +[b19n12:122873] [ 1] [0x3fffcfef1400] +[b19n12:122873] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n12:122873] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n03:147533] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n03:147533] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n07:45308] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n07:45308] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n07:45308] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n07:45308] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n07:45308] [14] [b07n05:176909] [ 0] [0x1000000504d8] +[b07n05:176909] [ 1] [0x3fffce6539a0] +[b07n05:176909] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n05:176909] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n09:170656] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n09:170656] [ 9] [a02n08:146117] *** Process received signal *** +[a02n08:146117] Signal: Segmentation fault (11) +[a02n08:146117] Signal code: (3) +[a02n08:146117] Failing at address: 0xb170000000000a0 +[a02n08:146117] [ 0] [0x1000000504d8] +[a02n08:146117] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n10:122756] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n10:122756] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n07:156490] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n03:135037] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n03:135037] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n03:135037] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n03:135037] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n03:135037] *** End of error message *** +[b04n07:02839] [ 0] [0x1000000504d8] +[b04n07:02839] [ 1] [0x3ffffc01c980] +[b04n07:02839] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n07:02839] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n14:126119] *** End of error message *** +[0x3ffffb39ae90] +[b18n04:136994] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n04:136994] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n04:136994] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n04:136994] [ 5] [0x1000000504d8] +[b15n11:134745] [ 1] [0x3fffdf117a30] +[b15n11:134745] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n11:134745] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n11:134745] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n02:125374] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n02:125374] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n02:125374] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n02:125374] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n08:118550] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n08:118550] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n08:118550] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n08:118550] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n03:68828] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n03:68828] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n03:68828] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n03:68828] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n03:68828] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n03:68828] [10] [b14n14:122440] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n01:123569] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n01:123569] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n12:145265] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n12:145265] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n12:145265] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n12:145265] [ 9] [b09n05:173819] [ 0] [0x1000000504d8] +[b09n05:173819] [ 1] [0x3ffff3e6f190] +[b09n05:173819] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n05:173819] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n05:173819] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n05:173819] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n05:173819] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n14:177221] *** End of error message *** +[a32n08:165284] [ 0] [0x1000000504d8] +[a32n08:165284] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n06:148311] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n06:148311] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n07:125451] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n04:136609] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n04:136609] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n04:136609] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n04:136609] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n13:137748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n13:137748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n13:137748] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n13:137748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n08:124114] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n08:124114] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n08:124114] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n08:124114] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n03:124681] [ 8] [0x3fffddfb14a0] +[b18n12:120404] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n12:120404] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n12:120404] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n12:120404] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n12:120404] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n12:120404] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n12:120404] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n12:120404] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n12:120404] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n12:120404] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n12:120404] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n12:120404] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n12:120404] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n12:120404] *** End of error message *** +[b18n11:128956] *** Process received signal *** +[b18n11:128956] Signal: Segmentation fault (11) +[b18n11:128956] Signal code: (3) +[b18n11:128956] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n07:163252] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n07:163252] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n05:05606] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n05:05606] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n05:05606] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n05:05606] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n05:05606] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n03:119831] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n03:119831] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n03:119831] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n03:119831] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n03:119831] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n07:156140] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n07:156140] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n18:121306] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n18:121306] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n18:121306] [11] [b22n14:116413] *** Process received signal *** +[b22n14:116413] Signal: Segmentation fault (11) +[b22n14:116413] Signal code: (3) +[b22n14:116413] Failing at address: 0x14170000000000a0 +[b22n14:116413] [ 0] [0x1000000504d8] +[b22n14:116413] [ 1] [0x3fffec3444c0] +[b22n14:116413] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n14:116413] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n14:116413] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n14:116413] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n14:116413] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n14:116413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n14:116413] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n14:116413] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n14:116413] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n14:116413] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n14:116413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n14:116413] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n14:116413] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n14:116413] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n16:129236] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n09:143897] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n09:143897] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n09:143897] [13] [b09n04:177594] *** Process received signal *** +[b09n04:177594] Signal: Segmentation fault (11) +[b09n04:177594] Signal code: (3) +[b09n04:177594] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n14:139843] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n14:139843] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n14:139843] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n14:139843] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n08:144855] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n08:144855] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n08:144855] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n08:144855] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n08:144855] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n04:117351] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n04:117351] [ 5] [a35n08:138178] *** Process received signal *** +[a35n08:138178] Signal: Segmentation fault (11) +[a35n08:138178] Signal code: (3) +[a35n08:138178] Failing at address: 0x5170000000000a0 +[a35n08:138178] [ 0] [0x1000000504d8] +[a35n08:138178] [ 1] [0x3fffd3f4b1a0] +[a35n08:138178] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n08:138178] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n08:138178] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n08:138178] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n08:138178] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n08:138178] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n08:138178] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n08:138178] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n08:138178] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n08:138178] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n08:138178] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n08:138178] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n08:138178] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n08:138178] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n12:180114] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n12:180114] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n12:180114] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n12:180114] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n12:180114] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n12:177494] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n12:177494] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n12:177494] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n12:177494] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n12:177494] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n05:53373] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n05:53373] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n09:166573] *** End of error message *** +[b19n18:122552] *** Process received signal *** +[b19n18:122552] Signal: Segmentation fault (11) +[b19n18:122552] Signal code: (3) +[b19n18:122552] Failing at address: 0xb170000000000a0 +[b19n18:122552] [ 0] [0x1000000504d8] +[b19n18:122552] [ 1] [0x3fffeae2d200] +[b19n18:122552] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n18:122552] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n07:149476] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n07:149476] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n07:149476] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n07:149476] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n05:121822] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n05:121822] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n05:121822] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n05:121822] [10] [a35n15:148163] *** Process received signal *** +[a35n15:148163] Signal: Segmentation fault (11) +[a35n15:148163] Signal code: (3) +[a35n15:148163] Failing at address: 0xb170000000000a0 +[a35n15:148163] [ 0] [0x1000000504d8] +[a35n15:148163] [ 1] [0x3ffffbf9a9a0] +[a35n15:148163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n15:148163] [ 3] [a35n13:138087] *** Process received signal *** +[a35n13:138087] Signal: Segmentation fault (11) +[a35n13:138087] Signal code: (3) +[a35n13:138087] Failing at address: 0xb170000000000a0 +[a35n13:138087] [ 0] [0x1000000504d8] +[a35n13:138087] [ 1] [0x3fffe2fdab80] +[a35n13:138087] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n13:138087] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n13:138087] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n16:130377] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n16:130377] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n16:130377] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n16:130377] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n16:130377] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n10:126191] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n10:126191] [ 5] [a32n16:147284] *** Process received signal *** +[a32n16:147284] Signal: Segmentation fault (11) +[a32n16:147284] Signal code: (3) +[a32n16:147284] Failing at address: 0xb170000000000a0 +[a32n16:147284] [ 0] [0x1000000504d8] +[a32n16:147284] [ 1] [a27n15:32400] *** Process received signal *** +[a27n15:32400] Signal: Segmentation fault (11) +[a27n15:32400] Signal code: (3) +[a27n15:32400] Failing at address: 0xb170000000000a0 +[a27n15:32400] [ 0] [0x1000000504d8] +[a27n15:32400] [ 1] [0x3fffd598dfa0] +[a27n15:32400] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n15:32400] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n12:164383] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n12:164383] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n12:164383] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n12:164383] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n04:168489] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n04:168489] *** End of error message *** +[a33n14:136136] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n14:136136] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n14:136136] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n01:163023] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n01:163023] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n01:163023] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n01:163023] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n15:119255] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n15:120061] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n17:125805] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n17:125805] [ 6] [a36n15:151249] [ 0] [0x1000000504d8] +[a36n15:151249] [ 1] [0x3fffc441be50] +[a36n15:151249] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n15:151249] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n11:163260] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n11:163260] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n11:163260] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n11:163260] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n11:163260] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n04:153962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n04:153962] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n04:153962] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n04:153962] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n04:153962] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n13:117508] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n13:117508] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n13:117508] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n13:117508] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n12:178717] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n10:134733] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n10:134733] [10] [a28n17:168162] [ 0] [0x1000000504d8] +[a28n17:168162] [ 1] [0x3fffcc5e7820] +[a28n17:168162] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n17:168162] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n14:150131] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n14:150131] [ 5] [0x3fffe3a22dc0] +[b21n14:119487] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n14:119487] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n14:119487] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n14:119487] [ 5] [0x1000000504d8] +[b20n04:126348] [ 1] [0x3fffe9637f50] +[b20n04:126348] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n04:126348] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n04:126348] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n04:126348] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n13:166924] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n13:166924] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n02:119531] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n02:119531] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n11:162470] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n11:162470] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n11:162470] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n11:162470] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n11:162470] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n11:162470] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n11:162470] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n11:162470] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n11:162470] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n11:162470] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n11:124432] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n11:124432] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n11:153416] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n11:153416] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n11:153416] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n11:153416] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n11:153416] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n13:168158] [ 6] [a31n03:158659] *** Process received signal *** +[a31n03:158659] Signal: Segmentation fault (11) +[a31n03:158659] Signal code: (3) +[a31n03:158659] Failing at address: 0x14170000000000a0 +[b09n16:173547] *** Process received signal *** +[b09n16:173547] Signal: Segmentation fault (11) +[b09n16:173547] Signal code: (3) +[b09n16:173547] Failing at address: 0x14170000000000a0 +[b09n16:173547] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n05:125613] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n05:125613] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n05:125613] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n05:125613] *** End of error message *** +[b11n09:161228] Signal: Segmentation fault (11) +[b11n09:161228] Signal code: (3) +[b11n09:161228] Failing at address: 0xb170000000000a0 +[b11n09:161228] [ 0] [0x1000000504d8] +[b11n09:161228] [ 1] [0x3ffff76d8310] +[b11n09:161228] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n09:161228] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n09:161228] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n09:161228] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n15:124801] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n15:124801] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n15:124801] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n15:124801] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n15:124801] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n15:124801] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n15:124801] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n15:124801] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n15:124801] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n18:140386] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n18:140386] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n18:140386] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n18:140386] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n18:140386] *** End of error message *** +[0x1000000504d8] +[b09n18:164799] [ 1] [0x3ffff5630310] +[b09n18:164799] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n18:164799] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n18:164799] [ 4] [b15n18:08431] *** Process received signal *** +[b15n18:08431] Signal: Segmentation fault (11) +[b15n18:08431] Signal code: (3) +[b15n18:08431] Failing at address: 0xb170000000000a0 +[b15n18:08431] [ 0] [0x1000000504d8] +[b15n18:08431] [ 1] [0x3fffe91f6520] +[b15n18:08431] [ 2] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n16:128439] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n16:128439] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n13:123883] [ 4] [b21n12:120127] *** Process received signal *** +[b21n12:120127] Signal: Segmentation fault (11) +[b21n12:120127] Signal code: (3) +[b21n12:120127] Failing at address: 0xb170000000000a0 +[b21n12:120127] [ 0] [0x1000000504d8] +[b21n12:120127] [ 1] [0x3fffc09506f0] +[b21n12:120127] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n12:120127] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n12:120127] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n05:164944] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n05:164944] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n05:164944] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n05:164944] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n05:164944] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n05:164944] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n05:164944] *** End of error message *** +[a27n08:54623] *** Process received signal *** +[a27n08:54623] Signal: Segmentation fault (11) +[a27n08:54623] Signal code: (3) +[a27n08:54623] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n17:129929] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n17:129929] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n17:129929] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n17:129929] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n17:129929] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n17:129929] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n18:131584] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n18:131584] [13] [a32n10:162608] [ 0] [0x1000000504d8] +[a32n10:162608] [ 1] [0x3fffd9b12b50] +[a32n10:162608] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n10:162608] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n10:162608] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n14:133064] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n14:133064] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n02:167599] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n04:166205] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n04:166205] [0x3ffff971d8f0] +[b18n17:124164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n17:124164] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n17:124164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n17:124164] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n17:124164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n08:131380] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n08:131380] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n08:131380] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n08:131380] [13] [b14n06:127272] *** Process received signal *** +[b14n06:127272] Signal: Segmentation fault (11) +[b14n06:127272] Signal code: (3) +[b14n06:127272] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n16:165925] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n16:165925] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n16:165925] [ 9] [0x1000000504d8] +[b21n08:116175] [ 1] [0x3fffc6682350] +[b21n08:116175] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n08:116175] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n08:116175] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n07:117358] [ 6] [ 0] [0x1000000504d8] +[b13n07:133376] [ 1] [0x3ffffaf400a0] +[b13n07:133376] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n07:133376] [ 3] [b15n12:132102] *** Process received signal *** +[b15n12:132102] Signal: Segmentation fault (11) +[b15n12:132102] Signal code: (3) +[b15n12:132102] Failing at address: 0xb170000000000a0 +[b15n12:132102] [ 0] [0x1000000504d8] +[b15n12:132102] [ 1] [0x3fffcce48910] +[b15n12:132102] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n12:132102] [ 3] [b05n02:176047] [ 0] [0x1000000504d8] +[b05n02:176047] [ 1] [0x3fffc7fe1640] +[b05n02:176047] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n02:176047] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n02:176047] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n02:176047] [ 5] [b17n16:127350] [ 0] [0x1000000504d8] +[b17n16:127350] [ 1] [0x3fffd1e93d60] +[b17n16:127350] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n16:127350] [ 3] [a31n06:167370] [ 0] [0x1000000504d8] +[a31n06:167370] [ 1] [0x3fffd3eed050] +[a31n06:167370] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n06:167370] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n06:167370] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n04:41672] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n04:41672] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n04:41672] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n04:41672] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n04:41672] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n04:41672] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n03:123614] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n11:170894] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n11:170894] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n11:170894] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n11:170894] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n11:170894] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n11:170894] [14] [a02n03:38499] *** Process received signal *** +[a02n03:38499] Signal: Segmentation fault (11) +[a02n03:38499] Signal code: (3) +[a02n03:38499] Failing at address: 0x14170000000000a0 +[a02n03:38499] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n01:128831] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n01:128831] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n01:128831] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n11:133968] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n11:133968] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n11:133968] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n11:133968] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n11:133968] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n11:133968] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n11:133968] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n11:133968] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n11:133968] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n11:133968] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n11:133968] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n11:133968] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n14:176069] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n16:157885] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n16:157885] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n16:157885] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n16:157885] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n16:157885] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n16:157885] *** End of error message *** +[b02n07:180191] *** Process received signal *** +[b02n07:180191] Signal: Segmentation fault (11) +[b02n07:180191] Signal code: (3) +[b02n07:180191] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n13:122030] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n13:114395] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n13:114395] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n16:116523] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n16:116523] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n16:116523] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n16:116523] [ 8] [b10n11:160350] *** Process received signal *** +[b10n11:160350] Signal: Segmentation fault (11) +[b10n11:160350] Signal code: (3) +[b10n11:160350] Failing at address: 0x14170000000000a0 +[b10n11:160350] [ 0] [0x1000000504d8] +[b10n11:160350] [ 1] [0x3fffe2a689a0] +[b10n11:160350] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n11:160350] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n10:142933] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n10:142933] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n10:142933] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n10:142933] [ 8] [b03n12:167674] *** Process received signal *** +[b03n12:167674] Signal: Segmentation fault (11) +[b03n12:167674] Signal code: (3) +[b03n12:167674] Failing at address: 0xb170000000000a0 +[b03n12:167674] [ 0] [0x1000000504d8] +[b03n12:167674] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n01:48802] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n01:48802] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n18:179696] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n18:179696] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n09:51333] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n09:51333] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n09:51333] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n09:51333] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n09:51333] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n09:51333] [13] [b19n09:123415] *** Process received signal *** +[b19n09:123415] Signal: Segmentation fault (11) +[b19n09:123415] Signal code: (3) +[b19n09:123415] Failing at address: 0xb170000000000a0 +[b19n09:123415] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n12:12765] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n02:06562] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n02:06562] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n02:06562] [13] [b23n07:118164] *** Process received signal *** +[b23n07:118164] Signal: Segmentation fault (11) +[b23n07:118164] Signal code: (3) +[b23n07:118164] Failing at address: 0xb170000000000a0 +[b23n07:118164] [ 0] [0x1000000504d8] +[b23n07:118164] [ 1] [0x3fffd816ce40] +[b23n07:118164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n07:118164] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n17:149906] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n15:155429] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n15:155429] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n15:177055] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n15:177055] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n15:177055] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n15:177055] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n06:126379] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n06:126379] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n06:126379] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n10:176310] *** End of error message *** +[b02n17:03843] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n14:128867] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b16n14:133636] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b16n14:133636] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b16n14:133636] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n05:135162] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n05:135162] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n17:15870] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n17:15870] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n17:15870] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n17:15870] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n14:169521] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n14:169521] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n14:169521] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n14:136658] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n14:136658] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n14:136658] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n14:136658] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n14:136658] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n14:136658] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n14:136658] [11] [b02n01:167301] *** Process received signal *** +[a33n01:141815] *** Process received signal *** +[a33n01:141815] Signal: Segmentation fault (11) +[a33n01:141815] Signal code: (3) +[a33n01:141815] Failing at address: 0x14170000000000a0 +[a33n01:141815] [ 0] [0x1000000504d8] +[a33n01:141815] [ 1] [a28n13:176364] [ 0] [0x1000000504d8] +[a28n13:176364] [ 1] [0x3fffdcc580a0] +[a28n13:176364] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n13:176364] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n06:01839] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n06:01839] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n06:01839] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n06:01839] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n08:08955] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n08:08955] *** End of error message *** +[b22n12:122966] [ 0] [0x1000000504d8] +[b22n12:122966] [ 1] [0x3fffc02b23e0] +[b22n12:122966] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n12:122966] [ 3] [a35n07:138205] *** Process received signal *** +[a35n07:138205] Signal: Segmentation fault (11) +[a35n07:138205] Signal code: (3) +[a35n07:138205] Failing at address: 0x5170000000000a0 +[a35n07:138205] [ 0] [0x1000000504d8] +[a35n07:138205] [ 1] [0x3fffe2abc3f0] +[a35n07:138205] [ 2] [b17n18:126906] *** Process received signal *** +[b17n18:126906] Signal: Segmentation fault (11) +[b17n18:126906] Signal code: (3) +[b17n18:126906] Failing at address: 0x14170000000000a0 +[b17n18:126906] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n06:143010] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n06:143010] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n06:143010] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n07:132086] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n07:132086] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n07:132086] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n08:147820] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n08:147820] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n08:147820] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n08:147820] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n08:147820] [ 8] [a33n04:169910] *** Process received signal *** +[a33n04:169910] Signal: Segmentation fault (11) +[a33n04:169910] Signal code: (3) +[a33n04:169910] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n07:153174] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n07:153174] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n07:153174] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n07:153174] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n07:153174] *** End of error message *** +[b19n07:131670] *** Process received signal *** +[b19n07:131670] Signal: Segmentation fault (11) +[b19n07:131670] Signal code: (3) +[b19n07:131670] Failing at address: 0xb170000000000a0 +[b19n07:131670] [ 0] [b10n02:164492] *** Process received signal *** +[b10n02:164492] Signal: Segmentation fault (11) +[b10n02:164492] Signal code: (3) +[b10n02:164492] Failing at address: 0xb170000000000a0 +[b10n02:164492] [ 0] [0x1000000504d8] +[b10n02:164492] [ 1] [0x3fffca6d3560] +[b10n02:164492] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n02:164492] [ 3] [b02n02:166743] *** Process received signal *** +[b02n02:166743] Signal: Segmentation fault (11) +[b02n02:166743] Signal code: (3) +[b02n02:166743] Failing at address: 0x14170000000000a0 +[a36n17:150931] [ 0] [0x1000000504d8] +[a36n17:150931] [ 1] [0x3fffde41cee0] +[a36n17:150931] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n17:150931] [ 3] [b12n11:137869] [ 0] [0x1000000504d8] +[b12n11:137869] [ 1] [0x3ffff25690c0] +[b12n11:137869] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n11:137869] [ 3] [0x1000000504d8] +[b02n16:01570] [ 1] [0x3fffc62f7180] +[b02n16:01570] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n16:01570] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n16:01570] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n16:01570] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n16:01570] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n01:128154] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n01:128154] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n01:128154] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n01:128154] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n03:10288] [ 6] [0x1000000504d8] +[b15n09:121751] [ 1] [0x3fffe0844150] +[b15n09:121751] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n09:121751] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n09:121751] [ 4] [b15n13:123706] Signal: Segmentation fault (11) +[b15n13:123706] Signal code: (3) +[b15n13:123706] Failing at address: 0xb170000000000a0 +[b15n13:123706] [ 0] [0x1000000504d8] +[b15n13:123706] [ 1] [0x3fffecb3f9c0] +[b15n13:123706] [ 2] [b11n11:152985] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n11:01171] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n11:01171] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n11:01171] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n11:01171] [13] [b09n11:171962] *** Process received signal *** +[b09n11:171962] Signal: Segmentation fault (11) +[b09n11:171962] Signal code: (3) +[b09n11:171962] Failing at address: 0xb170000000000a0 +[a27n18:28220] *** Process received signal *** +[a27n18:28220] Signal: Segmentation fault (11) +[a27n18:28220] Signal code: (3) +[a27n18:28220] Failing at address: 0xb170000000000a0 +[a27n18:28220] [ 0] [0x1000000504d8] +[a27n18:28220] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n08:174409] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n08:174409] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n08:174409] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n08:174409] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n08:174409] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n08:124722] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n08:124722] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n08:124722] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n01:130408] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n01:130408] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n01:130408] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n01:130408] [ 9] [0x1000000504d8] +[b01n09:173752] [ 1] [0x3fffd4299260] +[b01n09:173752] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n09:173752] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n09:173752] [ 4] [b11n14:161620] *** Process received signal *** +[b11n14:161620] Signal: Segmentation fault (11) +[b11n14:161620] Signal code: (3) +[b11n14:161620] Failing at address: 0x14170000000000a0 +[b11n14:161620] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n15:85942] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n15:85942] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n15:85942] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n10:130580] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n10:130580] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n10:130580] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n10:130580] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n10:130580] [14] [0x1000000504d8] +[b12n02:143965] [ 1] [0x3ffffead2b90] +[b12n02:143965] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n02:143965] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n02:143965] [ 4] [0x1000000504d8] +[b14n16:122492] [ 1] [0x3fffd3f1b0e0] +[b14n16:122492] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n16:122492] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n16:122492] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n16:153292] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n16:153292] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n16:153292] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n16:153292] [ 9] [a30n12:148378] *** Process received signal *** +[a30n12:148378] Signal: Segmentation fault (11) +[a30n12:148378] Signal code: (3) +[a30n12:148378] Failing at address: 0xb170000000000a0 +[a30n12:148378] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n06:122162] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n06:122162] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n06:122162] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n06:122162] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n06:122162] *** End of error message *** +[b05n12:11142] *** Process received signal *** +[b05n12:11142] Signal: Segmentation fault (11) +[b05n12:11142] Signal code: (3) +[b05n12:11142] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n01:173782] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n01:173782] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n01:173782] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n01:173782] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n01:173782] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n09:119545] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n09:119545] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n09:119545] [12] [a31n12:156915] *** Process received signal *** +[a31n12:156915] Signal: Segmentation fault (11) +[a31n12:156915] Signal code: (3) +[a31n12:156915] Failing at address: 0xb170000000000a0 +[a31n12:156915] [ 0] [0x1000000504d8] +[a31n12:156915] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n02:120188] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n02:120188] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n02:120188] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n02:120188] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n03:127629] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n03:127629] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n03:127629] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n03:127629] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n03:127629] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n09:123810] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n16:171916] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n16:171916] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n16:171916] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n16:171916] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n16:171916] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n16:171916] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n16:171916] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n16:171916] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n17:119580] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n17:119580] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n17:119580] [ 9] [b06n04:175997] [ 0] [0x1000000504d8] +[b06n04:175997] [ 1] [0x3fffdee47380] +[b06n04:175997] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n04:175997] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n04:175997] [ 4] [b12n01:145730] *** Process received signal *** +[b12n01:145730] Signal: Segmentation fault (11) +[b12n01:145730] Signal code: (3) +[b12n01:145730] Failing at address: 0x14170000000000a0 +[b12n01:145730] [ 0] [0x1000000504d8] +[b12n01:145730] [ 1] [a33n15:148582] [ 0] [0x1000000504d8] +[a33n15:148582] [ 1] [0x3fffe07ea110] +[a33n15:148582] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n15:148582] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n15:148582] [ 4] [b06n02:163319] *** Process received signal *** +[b06n02:163319] Signal: Segmentation fault (11) +[b06n02:163319] Signal code: (3) +[b06n02:163319] Failing at address: 0xb170000000000a0 +[b06n02:163319] [ 0] [0x1000000504d8] +[b06n02:163319] [ 1] [0x1000000504d8] +[b12n04:138156] [ 1] [0x3fffcab73f10] +[b12n04:138156] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n04:138156] [ 3] [0x1000000504d8] +[b17n07:125470] [ 1] [0x3fffc3e69300] +[b17n07:125470] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n07:125470] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n07:125470] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n07:125470] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n07:125470] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n07:125470] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n07:125470] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n08:163164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n08:163164] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n17:164562] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n10:122206] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n10:122206] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n08:121686] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n05:169785] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n03:168209] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n03:168209] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n03:168209] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n03:168209] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n03:168209] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n14:173892] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n14:173892] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n14:173892] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n14:173892] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n14:173892] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n14:173892] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n14:173892] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n14:173892] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n14:173892] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n14:173892] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n14:173892] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n14:173892] *** End of error message *** +[b14n07:127357] [ 0] [0x1000000504d8] +[b14n07:127357] [ 1] [0x3fffd53f65a0] +[b14n07:127357] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n07:127357] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n07:127357] [ 4] [0x3fffd1bd74a0] +[a31n08:156368] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n08:156368] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n08:156368] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n08:156368] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n08:156368] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n01:165163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n01:165163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n01:165163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n01:165163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n01:165163] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n12:115639] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n12:115639] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n04:169409] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n04:169409] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n04:169409] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n04:169409] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n04:169409] *** End of error message *** +[b07n13:165259] *** Process received signal *** +[b07n13:165259] Signal: Segmentation fault (11) +[b07n13:165259] Signal code: (3) +[b07n13:165259] Failing at address: 0x14170000000000a0 +[b07n13:165259] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n08:163099] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n08:163099] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n12:177119] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n12:177119] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n18:163110] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n18:163110] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n18:163110] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n18:163110] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n18:163110] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n18:163110] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n18:163110] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n11:147713] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n11:147713] [ 8] [ 0] [0x1000000504d8] +[b17n15:137004] [ 1] [0x3fffcd4f88b0] +[b17n15:137004] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n15:137004] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n15:137004] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n11:120238] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n11:120238] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n15:154382] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n15:154382] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n15:154382] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n15:154382] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n15:154382] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n02:149227] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n02:149227] *** End of error message *** +[b03n11:175593] *** Process received signal *** +[b03n11:175593] Signal: Segmentation fault (11) +[b03n11:175593] Signal code: (3) +[b03n11:175593] Failing at address: 0x14170000000000a0 +[b03n11:175593] [ 0] [a35n01:138006] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n11:02682] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n11:02682] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n17:146793] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n17:146793] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n17:146793] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n17:146793] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n17:126524] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n06:10226] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n12:140124] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n12:140124] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n12:140124] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n12:140124] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n12:140124] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n12:140124] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n12:140124] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n12:140124] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n12:140124] *** End of error message *** +[b10n03:168933] Signal: Segmentation fault (11) +[b10n03:168933] Signal code: (3) +[b10n03:168933] Failing at address: 0xb170000000000a0 +[b10n03:168933] [ 0] [0x1000000504d8] +[b10n03:168933] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n09:164364] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n09:164364] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n09:164364] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n15:123037] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n15:123037] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n14:163549] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n14:163549] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n03:135161] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n03:135161] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n03:135161] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n12:154345] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n12:154345] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n12:154345] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n12:154345] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n12:154345] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n12:11419] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n12:11419] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n12:11419] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n12:11419] *** End of error message *** +[b06n15:161753] *** Process received signal *** +[b06n15:161753] Signal: Segmentation fault (11) +[b06n15:161753] Signal code: (3) +[b06n15:161753] Failing at address: 0x5170000000000a0 +[b07n06:177007] *** Process received signal *** +[b07n06:177007] Signal: Segmentation fault (11) +[b07n06:177007] Signal code: (3) +[b07n06:177007] Failing at address: 0x14170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n02:178474] *** End of error message *** +[b18n03:136664] [ 0] [0x1000000504d8] +[b18n03:136664] [ 1] [0x3fffc0d530e0] +[b18n03:136664] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n03:136664] [ 3] [a26n01:65727] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n01:65727] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n01:65727] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n01:65727] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n01:65727] *** End of error message *** +[b23n03:125243] Signal: Segmentation fault (11) +[b23n03:125243] Signal code: (3) +[b23n03:125243] Failing at address: 0x14170000000000a0 +[b23n03:125243] [ 0] [0x1000000504d8] +[b23n03:125243] [ 1] [0x3ffff51b4140] +[b23n03:125243] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n03:125243] [ 3] [a32n06:154620] [ 0] [0x1000000504d8] +[a32n06:154620] [ 1] [0x3fffc26e6350] +[a32n06:154620] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n06:154620] [ 3] [b11n05:162568] *** Process received signal *** +[b11n05:162568] Signal: Segmentation fault (11) +[b11n05:162568] Signal code: (3) +[b11n05:162568] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n04:169706] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n04:169706] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n04:169706] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n04:169706] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n04:169706] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n04:169706] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n04:169706] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n04:169706] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n04:169706] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n03:01354] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n03:01354] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n03:01354] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n03:01354] [13] [a35n11:148672] *** Process received signal *** +[a35n11:148672] Signal: Segmentation fault (11) +[a35n11:148672] Signal code: (3) +[a35n11:148672] Failing at address: 0xb170000000000a0 +[a35n11:148672] [ 0] [0x1000000504d8] +[a35n11:148672] [ 1] [ 1] [0x3fffff7376b0] +[a33n09:147794] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n09:147794] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n09:147794] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n09:147794] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n02:122555] *** End of error message *** +[b14n17:119465] *** Process received signal *** +[b14n17:119465] Signal: Segmentation fault (11) +[b14n17:119465] Signal code: (3) +[b14n17:119465] Failing at address: 0xb170000000000a0 +[b14n17:119465] [ 0] [0x1000000504d8] +[b14n17:119465] [ 1] [0x3ffffec5a6a0] +[b14n17:119465] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n17:119465] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n10:161717] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n10:161717] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n10:161717] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n10:161717] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n10:161717] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n10:161717] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n02:123702] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n02:123702] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n17:162358] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n17:162358] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n17:162358] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n17:162358] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n17:162358] *** End of error message *** +[b18n06:133956] *** Process received signal *** +[b18n06:133956] Signal: Segmentation fault (11) +[b18n06:133956] Signal code: (3) +[b18n06:133956] Failing at address: 0xb170000000000a0 +[b18n06:133956] [ 0] [0x1000000504d8] +[b18n06:133956] [ 1] [0x3fffe7105aa0] +[b18n06:133956] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n06:133956] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n04:13079] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n05:135120] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n05:135120] [ 6] [a34n01:164961] *** Process received signal *** +[a34n01:164961] Signal: Segmentation fault (11) +[a34n01:164961] Signal code: (3) +[a34n01:164961] Failing at address: 0xb170000000000a0 +[a34n01:164961] [ 0] [0x1000000504d8] +[a34n01:164961] [ 1] [0x3ffff5104e30] +[a34n01:164961] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n01:164961] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n06:130541] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n17:163818] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n17:163818] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n17:163818] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n17:163818] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n17:163818] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n17:163818] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n03:151880] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n03:151880] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n03:151880] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n05:131263] [ 6] [a28n01:179260] *** Process received signal *** +[a28n01:179260] Signal: Segmentation fault (11) +[a28n01:179260] Signal code: (3) +[a28n01:179260] Failing at address: 0xb170000000000a0 +[a28n01:179260] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n05:154189] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n05:154189] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n05:154189] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n05:154189] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n05:154189] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n01:171785] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n01:171785] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n01:171785] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n01:171785] *** End of error message *** +[a36n02:147881] [ 0] [0x1000000504d8] +[a36n02:147881] [ 1] [0x3ffff1c33b60] +[a36n02:147881] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n02:147881] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n02:147881] [ 4] [b21n17:118571] *** Process received signal *** +[b21n17:118571] Signal: Segmentation fault (11) +[b21n17:118571] Signal code: (3) +[b21n17:118571] Failing at address: 0xb170000000000a0 +[b21n17:118571] [ 0] [a36n01:141344] *** Process received signal *** +[a36n01:141344] Signal: Segmentation fault (11) +[a36n01:141344] Signal code: (3) +[a36n01:141344] Failing at address: 0x5170000000000a0 +[a36n01:141344] [ 0] [0x1000000504d8] +[a36n01:141344] [ 1] [0x3fffe0f48de0] +[a36n01:141344] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n04:142254] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n04:142254] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n04:142254] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n04:142254] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n04:142254] *** End of error message *** +[0x1000000504d8] +[b01n08:172429] [ 1] [0x3fffc2f81ce0] +[b01n08:172429] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n08:172429] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n08:172429] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n18:118681] *** End of error message *** +[a36n03:153852] *** Process received signal *** +[a36n03:153852] Signal: Segmentation fault (11) +[a36n03:153852] Signal code: (3) +[a36n03:153852] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n16:121929] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n16:121929] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n16:121929] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n16:121929] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n16:121929] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n16:121929] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n10:164937] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n10:164937] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n10:164937] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n10:164937] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n10:164937] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n08:164749] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n06:162916] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n06:162916] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n11:164994] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n11:164994] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n11:164994] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n11:164994] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n04:04848] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n04:04848] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n04:04848] [ 8] [0x1000000504d8] +[b01n06:171500] [ 1] [0x3fffc9479000] +[b01n06:171500] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n06:171500] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n06:171500] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n03:130349] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n03:130349] [10] [b10n05:173078] *** Process received signal *** +[b10n05:173078] Signal: Segmentation fault (11) +[b10n05:173078] Signal code: (3) +[b10n05:173078] Failing at address: 0x14170000000000a0 +[b10n05:173078] [ 0] [0x1000000504d8] +[b10n05:173078] [ 1] [0x3fffdf864e80] +[b10n05:173078] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n05:173078] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n05:173078] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n05:173078] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n05:173078] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n05:173078] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n05:173078] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n05:173078] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n05:173078] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n05:173078] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n05:173078] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n05:173078] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n05:173078] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n05:173078] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n06:120732] [ 5] [0x1000000504d8] +[b14n02:139027] [ 1] [0x3fffea3a0490] +[b14n02:139027] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n02:139027] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n02:139027] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n02:139027] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n02:139027] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n02:139027] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n02:139027] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n06:163675] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n06:163675] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n06:163675] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n06:163675] [ 8] [b14n01:128002] [ 0] [0x1000000504d8] +[b14n01:128002] [ 1] [0x3ffff5fd58c0] +[b14n01:128002] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n01:128002] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n12:138735] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n12:138735] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n05:120329] [ 4] [b06n11:160796] *** Process received signal *** +[b06n11:160796] Signal: Segmentation fault (11) +[b06n11:160796] Signal code: (3) +[b06n11:160796] Failing at address: 0x5170000000000a0 +[b06n11:160796] [ 0] [0x1000000504d8] +[b06n11:160796] [ 1] [0x3fffc2ee6a00] +[b06n11:160796] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n11:160796] [ 3] [b19n17:123236] *** Process received signal *** +[b19n17:123236] Signal: Segmentation fault (11) +[b19n17:123236] Signal code: (3) +[b19n17:123236] Failing at address: 0x14170000000000a0 +[b19n17:123236] [ 0] [0x1000000504d8] +[b19n17:123236] [ 1] [0x3fffdf08e770] +[b19n17:123236] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n16:164550] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n16:164550] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n16:164550] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n16:164550] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n04:126857] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n04:126857] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n04:126857] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n05:126779] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n05:126779] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n05:126779] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n05:126779] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n05:126779] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n01:165122] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n01:165122] [ 6] [b15n07:124349] [ 0] [0x1000000504d8] +[b15n07:124349] [ 1] [0x3ffff505f1f0] +[b15n07:124349] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n07:124349] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n09:155760] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n09:155760] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n09:155760] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n13:153677] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n13:153677] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n13:153677] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n13:153677] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n13:10196] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n13:10196] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n13:10196] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n13:10196] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n13:10196] [14] [0x1000000504d8] +[a29n06:157787] [ 1] [0x3fffccfec150] +[a29n06:157787] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n06:157787] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n06:157787] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n06:157787] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n06:157787] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n06:157787] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n06:157787] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n11:117472] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n11:117472] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n11:117472] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n11:117472] [ 8] [b11n03:154292] *** Process received signal *** +[b11n03:154292] Signal: Segmentation fault (11) +[b11n03:154292] Signal code: (3) +[b11n03:154292] Failing at address: 0xb170000000000a0 +[b11n03:154292] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n10:16973] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n06:161724] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n06:161724] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n06:161724] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n11:03985] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n11:120584] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n11:120584] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n06:117528] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n06:117528] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n17:176412] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n07:160800] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n07:160800] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n07:160800] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n07:160800] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n14:178325] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n14:178325] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n14:178325] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n14:178325] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n14:178325] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n17:168155] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n17:168155] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n17:168155] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n17:168155] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n17:168155] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n17:168155] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n04:158632] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n04:158632] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n04:158632] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n04:158632] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n18:179170] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n18:179170] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n18:179170] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n18:179170] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n18:179170] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n12:178599] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n12:178599] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n12:178599] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n12:178599] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n12:178599] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n13:123514] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n13:123514] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n13:123514] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n13:123514] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n09:119632] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n13:171107] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n04:145748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n04:145748] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n11:134519] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n11:134519] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n11:134519] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n11:134519] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n11:134519] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n11:134519] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n11:134519] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n04:44758] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n04:44758] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n05:139437] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n05:139437] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n05:139437] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n07:168961] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n10:11492] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n10:11492] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n10:11492] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n03:06823] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n03:06823] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n03:06823] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n12:123561] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n05:139913] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n05:139913] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n05:139913] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n05:139913] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n01:163390] [ 8] [b22n10:118389] *** Process received signal *** +[a29n05:174779] *** Process received signal *** +[a29n05:174779] Signal: Segmentation fault (11) +[a29n05:174779] Signal code: (3) +[a29n05:174779] Failing at address: 0xb170000000000a0 +[a29n05:174779] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n10:169210] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n10:169210] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n10:169210] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n10:169210] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n10:169210] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n10:169210] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n10:169210] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n13:123052] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n13:123052] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n04:03147] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n01:96749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n01:96749] *** End of error message *** +[a30n13:154233] *** Process received signal *** +[a30n13:154233] Signal: Segmentation fault (11) +[a30n13:154233] Signal code: (3) +[a30n13:154233] Failing at address: 0x14170000000000a0 +[a28n14:08399] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n12:160962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n12:160962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n12:160962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n12:160962] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n09:12245] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n09:12245] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n05:120228] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n05:120228] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n05:120228] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n03:172650] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n03:172650] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n13:133627] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n11:130804] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n11:130804] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n11:130804] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n08:176683] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n08:176683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n08:176683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n01:170766] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n01:170766] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n01:170766] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n01:170766] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n01:170766] [14] [b11n16:154182] *** Process received signal *** +[b11n16:154182] Signal: Segmentation fault (11) +[b11n16:154182] Signal code: (3) +[b11n16:154182] Failing at address: 0xb170000000000a0 +[b11n16:154182] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n15:120129] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n06:179454] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n10:140414] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n10:140414] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n10:140414] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n10:140414] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n10:140414] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n18:128544] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n18:128544] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n18:128544] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n18:128544] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n18:128544] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n14:152747] [10] [a33n11:156151] [ 0] [0x1000000504d8] +[a33n11:156151] [ 1] [0x3fffc9c4f550] +[a33n11:156151] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n11:156151] [ 3] [b20n02:122177] *** Process received signal *** +[b20n02:122177] Signal: Segmentation fault (11) +[b20n02:122177] Signal code: (3) +[b20n02:122177] Failing at address: 0xb170000000000a0 +[b20n02:122177] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n08:159025] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n08:159025] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n08:159025] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n08:159025] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n08:159025] *** End of error message *** +[0x3ffffa6e9060] +[b13n15:139750] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n15:139750] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n15:139750] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n15:139750] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n15:139750] [ 6] [a34n07:165088] *** Process received signal *** +[a34n07:165088] Signal: Segmentation fault (11) +[a34n07:165088] Signal code: (3) +[a34n07:165088] Failing at address: 0xb170000000000a0 +[a34n07:165088] [ 0] [0x1000000504d8] +[a34n07:165088] [ 1] [0x3fffcea0b190] +[a34n07:165088] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n07:165088] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n10:10925] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n10:10925] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n10:155400] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n10:155400] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n10:155400] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n10:155400] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n10:155400] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n10:155400] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n15:122084] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n15:122084] *** End of error message *** +[0x1000000504d8] +[b09n10:169681] [ 1] [0x3fffde9c56c0] +[b09n10:169681] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n10:169681] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n10:169681] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n10:169681] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n10:169681] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n10:169681] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n10:169681] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n10:169681] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n10:169681] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n10:169681] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n10:169681] [12] [0x1000000504d8] +[b14n08:127442] [ 1] [0x3fffe8ffc5e0] +[b14n08:127442] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n08:127442] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n08:127442] [ 4] [a02n10:135854] *** Process received signal *** +[a02n10:135854] Signal: Segmentation fault (11) +[a02n10:135854] Signal code: (3) +[a02n10:135854] Failing at address: 0x14170000000000a0 +[a02n10:135854] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n02:74431] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n02:74431] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n18:10724] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n18:10724] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n16:124961] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n16:124961] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n03:164495] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n03:164495] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n03:164495] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n03:164495] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n01:121745] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n01:121745] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n01:121745] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n01:121745] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n01:121745] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n07:172879] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n07:172879] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n07:172879] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n10:120977] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n10:120977] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n10:120977] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n10:120977] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n10:120977] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n10:120977] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n12:160190] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n12:160190] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n12:160190] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n12:160190] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n12:160190] [14] [a31n13:155848] [ 0] [0x1000000504d8] +[a31n13:155848] [ 1] [0x3ffffa86b510] +[a31n13:155848] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n13:155848] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n13:155848] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n13:155848] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n13:155848] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n13:155848] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n13:155848] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n13:155848] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n13:155848] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n13:155848] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n13:155848] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n13:155848] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n13:155848] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n13:155848] *** End of error message *** +[b19n12:122874] *** Process received signal *** +[b19n12:122874] Signal: Segmentation fault (11) +[b19n12:122874] Signal code: (3) +[b19n12:122874] Failing at address: 0x14170000000000a0 +[b19n12:122874] [ 0] [0x1000000504d8] +[b19n12:122874] [ 1] [0x3fffc1712ed0] +[b19n12:122874] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n12:122874] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n04:130744] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n04:130744] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n04:130744] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n04:130744] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n02:125373] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n02:125373] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n02:125373] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n02:125373] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n09:170656] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n09:170656] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n09:170656] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n07:45309] *** End of error message *** +[a02n08:146118] *** Process received signal *** +[a02n08:146118] Signal: Segmentation fault (11) +[a02n08:146118] Signal code: (3) +[a02n08:146118] Failing at address: 0xb170000000000a0 +[a02n08:146118] [ 0] [0x1000000504d8] +[a02n08:146118] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n05:176909] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n10:122757] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n07:156489] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n03:135038] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n03:135038] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n03:135038] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n07:02839] [ 4] [b17n14:126121] *** Process received signal *** +[b17n14:126121] Signal: Segmentation fault (11) +[b17n14:126121] Signal code: (3) +[b17n14:126121] Failing at address: 0x14170000000000a0 +[b17n14:126121] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n03:147534] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n03:147534] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n03:147534] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n03:147534] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n03:147534] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n03:147534] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n03:147534] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n03:147534] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n03:147534] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n03:147534] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n11:134746] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n11:134746] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n04:136993] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n04:136993] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n04:136993] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n03:68829] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n03:68829] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n03:68829] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n03:68829] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n03:68829] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n03:68829] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n03:68829] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n08:118550] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n08:118550] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n08:118550] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n08:118550] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n14:122441] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n14:122441] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n14:122441] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n14:122441] [ 8] [0x3fffdadf51c0] +[b09n05:173820] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n05:173820] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n05:173820] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n05:173820] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n05:173820] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n05:173820] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n05:173820] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n01:123570] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n01:123570] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n01:123570] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n01:123570] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n01:123570] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n01:123570] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n12:145266] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n12:145266] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n12:145266] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n12:145266] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n03:119830] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n03:119830] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n03:119830] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n03:119830] [14] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n08:165285] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n08:165285] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n08:165285] [ 5] [b18n07:125447] *** Process received signal *** +[b18n07:125447] Signal: Segmentation fault (11) +[b18n07:125447] Signal code: (3) +[b18n07:125447] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n08:124114] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n08:124114] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n08:124114] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n08:124114] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n08:124114] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n03:124681] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n13:137747] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n13:137747] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n13:137747] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n13:137747] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n13:137747] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n13:137747] *** End of error message *** +[b18n11:128957] *** Process received signal *** +[b18n11:128957] Signal: Segmentation fault (11) +[b18n11:128957] Signal code: (3) +[b18n11:128957] Failing at address: 0x14170000000000a0 +[b18n11:128957] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n05:05605] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n05:05605] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n05:05605] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n05:05605] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n07:163252] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n07:163252] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n07:163252] [14] [b11n07:156142] *** Process received signal *** +[b11n07:156142] Signal: Segmentation fault (11) +[b11n07:156142] Signal code: (3) +[b11n07:156142] Failing at address: 0xb170000000000a0 +[b11n07:156142] [ 0] [0x1000000504d8] +[b11n07:156142] [ 1] [0x3ffff9960480] +[b11n07:156142] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n07:156142] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n07:156142] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n07:156142] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n18:121306] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n18:121306] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n18:121306] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n14:177222] [10] [b09n04:177595] *** Process received signal *** +[b09n04:177595] Signal: Segmentation fault (11) +[b09n04:177595] Signal code: (3) +[b09n04:177595] Failing at address: 0xb170000000000a0 +[b09n04:177595] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n09:143897] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n09:143897] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n14:139842] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n14:139842] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n14:139842] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n14:139842] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n14:139842] [13] [b22n14:116414] *** Process received signal *** +[b22n14:116414] Signal: Segmentation fault (11) +[b22n14:116414] Signal code: (3) +[b22n14:116414] Failing at address: 0x14170000000000a0 +[b22n14:116414] [ 0] [0x1000000504d8] +[b22n14:116414] [ 1] [0x3fffc63648c0] +[b22n14:116414] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n14:116414] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n14:116414] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n14:116414] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n14:116414] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n14:116414] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n14:116414] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n14:116414] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n14:116414] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n14:116414] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n14:116414] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n14:116414] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n14:116414] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n14:116414] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n16:129236] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n04:117352] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n04:117352] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n04:117352] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n04:117352] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n08:144854] *** End of error message *** +[b02n09:166571] *** Process received signal *** +[b02n09:166571] Signal: Segmentation fault (11) +[b02n09:166571] Signal code: (3) +[b02n09:166571] Failing at address: 0x14170000000000a0 +[b02n09:166571] [ 0] [0x1000000504d8] +[b02n09:166571] [ 1] [0x3fffc6df5200] +[b02n09:166571] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n09:166571] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n09:166571] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n09:166571] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n09:166571] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n09:166571] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n09:166571] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n09:166571] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n09:166571] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n09:166571] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n09:166571] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n09:166571] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n09:166571] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n09:166571] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n18:122551] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n18:122551] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n07:149475] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n07:149475] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n07:149475] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n07:149475] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n07:149475] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n13:138088] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n13:138088] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n13:138088] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n05:121821] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n05:121821] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n05:121821] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n16:130376] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n16:130376] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n16:130376] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n10:126191] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n10:126191] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n10:126191] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n10:126191] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n10:126191] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n10:126191] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n10:126191] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n10:126191] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n10:126191] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n10:126191] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n15:148163] [ 4] [0x3ffff37e96a0] +[a32n16:147284] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n16:147284] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n16:147284] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n15:32398] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n15:32398] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n15:32398] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n15:32398] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n15:32398] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n15:32398] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n12:164382] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n12:164382] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n12:164382] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n12:164382] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n12:164382] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n12:164382] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n14:136136] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n01:163023] *** End of error message *** +[b02n04:168488] *** Process received signal *** +[b02n04:168488] Signal: Segmentation fault (11) +[b02n04:168488] Signal code: (3) +[b02n04:168488] Failing at address: 0xb170000000000a0 +[b02n04:168488] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n15:119254] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n15:119254] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n15:119254] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n15:119254] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n11:163261] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n11:163261] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n11:163261] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n11:163261] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n11:163261] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n04:153963] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n04:153963] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n15:151248] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n15:151248] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n15:151248] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n17:125805] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n17:125805] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n17:125805] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n15:120061] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n15:120061] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n15:120061] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n15:120061] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n15:120061] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n15:120061] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n15:120061] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n15:120061] *** End of error message *** +[b09n12:178716] *** Process received signal *** +[b09n12:178716] Signal: Segmentation fault (11) +[b09n12:178716] Signal code: (3) +[b09n12:178716] Failing at address: 0x14170000000000a0 +[b09n12:178716] [ 0] [0x1000000504d8] +[b09n12:178716] [ 1] [0x3fffc6307b30] +[b09n12:178716] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n12:178716] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n13:117507] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n13:117507] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n13:117507] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n13:117507] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n10:134734] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n10:134734] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n10:134734] [13] [a28n17:168163] *** Process received signal *** +[a28n17:168163] Signal: Segmentation fault (11) +[a28n17:168163] Signal code: (3) +[a28n17:168163] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n14:150131] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n14:119487] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n14:119487] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n14:119487] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n04:126347] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n04:126347] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n04:126347] [ 6] [b06n13:166922] *** Process received signal *** +[b06n13:166922] Signal: Segmentation fault (11) +[b06n13:166922] Signal code: (3) +[b06n13:166922] Failing at address: 0x14170000000000a0 +[b06n13:166922] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n02:119531] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n11:124432] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n11:124432] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n11:124432] [14] [a33n13:168159] *** Process received signal *** +[a33n13:168159] Signal: Segmentation fault (11) +[a33n13:168159] Signal code: (3) +[a33n13:168159] Failing at address: 0xb170000000000a0 +[a33n13:168159] [ 0] [0x1000000504d8] +[a33n13:168159] [ 1] [0x3ffff1013e70] +[a33n13:168159] [ 2] [a31n03:158658] *** Process received signal *** +[a31n03:158658] Signal: Segmentation fault (11) +[a31n03:158658] Signal code: (3) +[a31n03:158658] Failing at address: 0x14170000000000a0 +[a31n03:158658] [ 0] [0x1000000504d8] +[a31n03:158658] [ 1] [0x3fffe5557980] +[a31n03:158658] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n03:158658] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n11:153415] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n11:162469] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n11:162469] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n11:162469] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n11:162469] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n11:162469] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n11:162469] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n09:161229] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n09:161229] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n09:161229] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n09:161229] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n09:161229] [ 8] [0x1000000504d8] +[b09n16:173547] [ 1] [0x3fffd0793b80] +[b09n16:173547] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n16:173547] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n18:140387] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n18:140387] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n18:140387] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n18:140387] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n18:140387] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n15:124802] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n15:124802] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n15:124802] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n15:124802] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n15:124802] [13] [0x1000000504d8] +[b09n18:164800] [ 1] [0x3fffcf92c560] +[b09n18:164800] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n18:164800] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n18:164800] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n18:164800] [ 5] [b13n16:128442] *** Process received signal *** +[b13n16:128442] Signal: Segmentation fault (11) +[b13n16:128442] Signal code: (3) +[b13n16:128442] Failing at address: 0x14170000000000a0 +[0x1000000504d8] +[b15n18:08430] [ 1] [0x3fffdfc6e540] +[b15n18:08430] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n18:08430] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n18:08430] [ 4] [0x3ffffc1eeaa0] +[b19n13:123884] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n13:123884] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n13:123884] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n13:123884] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n12:120128] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n12:120128] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n12:120128] [13] [b02n05:164942] *** Process received signal *** +[b02n05:164942] Signal: Segmentation fault (11) +[b02n05:164942] Signal code: (3) +[b02n05:164942] Failing at address: 0xb170000000000a0 +[b02n05:164942] [ 0] [a27n08:54624] *** Process received signal *** +[a27n08:54624] Signal: Segmentation fault (11) +[a27n08:54624] Signal code: (3) +[a27n08:54624] Failing at address: 0x14170000000000a0 +[a27n08:54624] [ 0] [0x1000000504d8] +[a27n08:54624] [ 1] [0x3fffd5625330] +[a27n08:54624] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n08:54624] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n18:131584] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n17:129930] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n17:129930] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n17:129930] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n17:129930] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n17:129930] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n10:162608] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n10:162608] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n14:133064] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n14:133064] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n14:133064] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n14:133064] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n02:167598] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n02:167598] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n04:166204] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n04:166204] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n04:166204] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n04:166204] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n08:131379] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n08:131379] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n08:131379] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n08:131379] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n08:116175] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n08:116175] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n17:124163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n17:124163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n17:124163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n17:124163] [10] [b14n06:127273] *** Process received signal *** +[b14n06:127273] Signal: Segmentation fault (11) +[b14n06:127273] Signal code: (3) +[b14n06:127273] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n16:165925] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n16:165925] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n07:117358] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n07:117358] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n07:117358] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n07:133376] [ 4] [b15n12:132103] *** Process received signal *** +[b15n12:132103] Signal: Segmentation fault (11) +[b15n12:132103] Signal code: (3) +[b15n12:132103] Failing at address: 0xb170000000000a0 +[b15n12:132103] [ 0] [0x1000000504d8] +[b15n12:132103] [ 1] [0x3fffc62eeef0] +[b15n12:132103] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n12:132103] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n12:132103] [ 4] [b05n02:176048] *** Process received signal *** +[b05n02:176048] Signal: Segmentation fault (11) +[b05n02:176048] Signal code: (3) +[b05n02:176048] Failing at address: 0xb170000000000a0 +[b05n02:176048] [ 0] [0x1000000504d8] +[b05n02:176048] [ 1] [0x3fffd5030160] +[b05n02:176048] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n02:176048] [ 3] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n06:167371] [ 3] [b19n03:123617] *** Process received signal *** +[b19n03:123617] Signal: Segmentation fault (11) +[b19n03:123617] Signal code: (3) +[b19n03:123617] Failing at address: 0x14170000000000a0 +[a02n03:38500] *** Process received signal *** +[a02n03:38500] Signal: Segmentation fault (11) +[a02n03:38500] Signal code: (3) +[a02n03:38500] Failing at address: 0x14170000000000a0 +[a02n03:38500] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n11:170894] *** End of error message *** +[b17n16:127349] *** Process received signal *** +[b17n16:127349] Signal: Segmentation fault (11) +[b17n16:127349] Signal code: (3) +[b17n16:127349] Failing at address: 0x14170000000000a0 +[b17n16:127349] [ 0] [0x1000000504d8] +[b17n16:127349] [ 1] [0x3fffe34b1260] +[b17n16:127349] [ 2] [a02n04:41668] *** Process received signal *** +[a02n04:41668] Signal: Segmentation fault (11) +[a02n04:41668] Signal code: (3) +[a02n04:41668] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n11:160350] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n11:160350] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n10:142932] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n10:142932] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n10:142932] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n10:142932] [10] [a29n16:157886] *** Process received signal *** +[a29n16:157886] Signal: Segmentation fault (11) +[a29n16:157886] Signal code: (3) +[a29n16:157886] Failing at address: 0x14170000000000a0 +[a29n16:157886] [ 0] [0x1000000504d8] +[a29n16:157886] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n13:114394] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n13:114394] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n13:114394] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n13:114394] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n01:128830] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n01:128830] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n01:128830] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n01:128830] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n01:128830] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n16:116522] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n16:116522] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n16:116522] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n16:116522] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n14:176069] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n14:176069] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n14:176069] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n14:176069] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n14:176069] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n11:133969] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n11:133969] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n11:133969] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n11:133969] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n11:133969] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n11:133969] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n11:133969] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n11:133969] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n11:133969] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n11:133969] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n11:133969] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n11:133969] *** End of error message *** +[b02n07:180191] [ 0] [0x1000000504d8] +[b02n07:180191] [ 1] [0x3fffcb7fc1d0] +[b02n07:180191] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n07:180191] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n07:180191] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n18:179696] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n18:179696] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n18:179696] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n18:179696] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n13:122030] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n13:122030] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n13:122030] [ 9] [0x1000000504d8] +[b19n09:123415] [ 1] [0x3fffcbe8ee20] +[b19n09:123415] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n09:123415] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n09:123415] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n01:48803] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n01:48803] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n01:48803] [ 8] [0x1000000504d8] +[b03n12:167675] [ 1] [0x3fffe0c76440] +[b03n12:167675] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n12:167675] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n12:167675] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n02:06563] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n02:06563] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n02:06563] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n02:06563] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n02:06563] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n09:51334] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n09:51334] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n09:51334] *** End of error message *** +[b08n12:12762] *** Process received signal *** +[b08n12:12762] Signal: Segmentation fault (11) +[b08n12:12762] Signal code: (3) +[b08n12:12762] Failing at address: 0xb170000000000a0 +[b08n12:12762] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n15:155430] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n07:118163] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n15:177055] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n15:177055] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n15:177055] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n15:177055] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n17:149906] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n06:126379] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n06:126379] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n14:128868] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n14:128868] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n14:128868] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n14:128868] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n17:03843] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n14:133635] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n14:133635] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n14:133635] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n14:133635] [13] [b22n12:122967] [ 0] [0x1000000504d8] +[b22n12:122967] [ 1] [0x3fffcc0a3860] +[b22n12:122967] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n12:122967] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n12:122967] [ 4] [b20n07:132087] Signal: Segmentation fault (11) +[b20n07:132087] Signal code: (3) +[b20n07:132087] Failing at address: 0xb170000000000a0 +[b20n07:132087] [ 0] [0x1000000504d8] +[b20n07:132087] [ 1] [0x3fffc2b476e0] +[b20n07:132087] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n07:132087] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n14:169520] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n14:169520] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n14:169520] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n14:169520] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n14:169520] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n17:15871] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n17:15871] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n17:15871] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n17:15871] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n17:15871] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n17:15871] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n14:136658] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n14:136658] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n14:136658] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n14:136658] *** End of error message *** +[b02n01:167301] Signal: Segmentation fault (11) +[b02n01:167301] Signal code: (3) +[b02n01:167301] Failing at address: 0x14170000000000a0 +[b02n01:167301] [ 0] [0x1000000504d8] +[b02n01:167301] [ 1] [0x3fffd048f000] +[b02n01:167301] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n01:167301] [ 3] [ 0] [0x1000000504d8] +[a33n01:141814] [ 1] [0x3fffe558abf0] +[a33n01:141814] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n01:141814] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n01:141814] [ 4] [b05n08:08953] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n06:01839] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n06:01839] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n06:01839] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n13:176363] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n13:176363] [ 5] [a28n10:176306] *** Process received signal *** +[a28n10:176306] Signal: Segmentation fault (11) +[a28n10:176306] Signal code: (3) +[a28n10:176306] Failing at address: 0xb170000000000a0 +[b17n18:126907] *** Process received signal *** +[b17n18:126907] Signal: Segmentation fault (11) +[b17n18:126907] Signal code: (3) +[b17n18:126907] Failing at address: 0x14170000000000a0 +[b17n18:126907] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n06:143009] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n06:143009] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n06:143009] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n06:143009] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n06:143009] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n06:143009] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n07:138204] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n07:138204] [ 5] [0x3fffce630af0] +[a33n08:147819] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n08:147819] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n08:147819] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n08:147819] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n08:147819] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n08:147819] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n08:147819] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n05:135163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n05:135163] [ 6] [a33n04:169910] [ 0] [0x1000000504d8] +[a33n04:169910] [ 1] [0x3fffc5ad99f0] +[a33n04:169910] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n04:169910] [ 3] [0x1000000504d8] +[b10n02:164491] [ 1] [0x3fffdcdabfb0] +[b10n02:164491] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n02:164491] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n02:164491] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n07:153175] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n07:153175] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n07:153175] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n07:153175] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n17:150931] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n17:150931] [ 5] [0x1000000504d8] +[b19n07:131670] [ 1] [0x3fffc50a7810] +[b19n07:131670] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n07:131670] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n07:131670] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n11:137870] [ 4] [b02n02:166744] *** Process received signal *** +[b02n02:166744] Signal: Segmentation fault (11) +[b02n02:166744] Signal code: (3) +[b02n02:166744] Failing at address: 0x14170000000000a0 +[b02n02:166744] [ 0] [0x1000000504d8] +[b02n02:166744] [ 1] [0x3fffffad3fc0] +[b02n02:166744] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n02:166744] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n02:166744] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n16:01569] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n16:01569] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n01:128154] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n01:128154] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n01:128154] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n03:10289] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n03:10289] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n03:10289] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n03:10289] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n09:121751] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n09:121751] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n09:121751] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n09:121751] [ 8] [0x3fffe129b0b0] +[b15n13:123705] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n13:123705] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n13:123705] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n11:152985] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n11:01171] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n11:01171] *** End of error message *** +[b09n11:171961] *** Process received signal *** +[b09n11:171961] Signal: Segmentation fault (11) +[b09n11:171961] Signal code: (3) +[b09n11:171961] Failing at address: 0x14170000000000a0 +[b09n11:171961] [ 0] [0x1000000504d8] +[b09n11:171961] [ 1] [0x3fffe1580670] +[b09n11:171961] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n11:171961] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n01:173781] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n01:173781] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n01:173781] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n01:173781] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n08:174408] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n01:130409] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n01:130409] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n01:130409] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n01:130409] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n08:124723] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n08:124723] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n08:124723] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n09:173753] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n09:173753] [ 6] [b11n14:161619] [ 0] [0x1000000504d8] +[b11n14:161619] [ 1] [0x3fffde2cd130] +[b11n14:161619] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n14:161619] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n14:161619] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n15:85943] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n15:85943] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n15:85943] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n10:130579] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n10:130579] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n10:130579] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n10:130579] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n10:130579] [14] [b14n16:122491] [ 0] [0x1000000504d8] +[b14n16:122491] [ 1] [0x3fffdc2a5b00] +[b14n16:122491] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n16:122491] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n16:122491] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n02:143965] [ 5] [a30n16:153291] *** End of error message *** +[0x1000000504d8] +[a30n12:148378] [ 1] [0x3ffff17a91c0] +[a30n12:148378] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n12:148378] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n12:148378] [ 4] [b05n12:11143] *** Process received signal *** +[b05n12:11143] Signal: Segmentation fault (11) +[b05n12:11143] Signal code: (3) +[b05n12:11143] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n09:119546] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n09:119546] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n09:119546] [11] [0x1000000504d8] +[a27n18:28219] [ 1] [0x3fffc9bf41f0] +[a27n18:28219] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n18:28219] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n18:28219] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n18:28219] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n18:28219] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n18:28219] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n18:28219] [ 8] [ 1] [0x3ffff74aede0] +[a31n12:156914] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n12:156914] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n12:156914] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n02:120189] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n02:120189] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n02:120189] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n02:120189] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n02:120189] [10] [b18n09:123810] Signal: Segmentation fault (11) +[b18n09:123810] Signal code: (3) +[b18n09:123810] Failing at address: 0x14170000000000a0 +[b18n09:123810] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n03:127628] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n03:127628] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n03:127628] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n03:127628] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n03:127628] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n04:175996] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n04:175996] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n17:119579] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n17:119579] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n17:119579] [ 6] [b12n01:145731] *** Process received signal *** +[b12n01:145731] Signal: Segmentation fault (11) +[b12n01:145731] Signal code: (3) +[b12n01:145731] Failing at address: 0xb170000000000a0 +[b12n01:145731] [ 0] [0x1000000504d8] +[b12n01:145731] [ 1] [0x3fffff91a230] +[b12n01:145731] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n01:145731] [ 3] [0x1000000504d8] +[a33n15:148583] [ 1] [0x3ffffa803880] +[a33n15:148583] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n15:148583] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n15:148583] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n15:148583] [ 5] [0x3fffd5e7d900] +[b06n02:163319] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n02:163319] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n02:163319] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n07:125471] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n07:125471] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n07:125471] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n07:125471] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n07:125471] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n07:125471] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n07:125471] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n07:125471] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n07:125471] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n04:138156] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n08:163163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n08:163163] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n10:122206] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n10:122206] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n17:164563] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n17:164563] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n11:147713] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n11:147713] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n03:168208] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n18:163109] *** End of error message *** +[b06n14:173893] *** Process received signal *** +[b06n14:173893] Signal: Segmentation fault (11) +[b06n14:173893] Signal code: (3) +[b06n14:173893] Failing at address: 0x5170000000000a0 +[b06n14:173893] [ 0] [0x1000000504d8] +[b06n14:173893] [ 1] [0x3ffffcf68300] +[b06n14:173893] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n14:173893] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n14:173893] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n14:173893] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n14:173893] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n14:173893] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n14:173893] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n14:173893] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n14:173893] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n14:173893] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n14:173893] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n14:173893] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n14:173893] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n14:173893] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n07:127357] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n07:127357] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n08:156368] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n08:156368] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n12:115639] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n12:115639] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n01:165162] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n01:165162] *** End of error message *** +[a32n13:163192] *** Process received signal *** +[a32n13:163192] Signal: Segmentation fault (11) +[a32n13:163192] Signal code: (3) +[a32n13:163192] Failing at address: 0xb170000000000a0 +[a32n13:163192] [ 0] [0x1000000504d8] +[a32n13:163192] [ 1] [0x3fffcf4644b0] +[a32n13:163192] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n13:163192] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n13:163192] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n13:163192] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n13:163192] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n13:163192] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n13:163192] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n13:163192] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n13:163192] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n13:163192] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n13:163192] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n13:163192] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n13:163192] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n13:163192] *** End of error message *** +[b07n13:165258] *** Process received signal *** +[b07n13:165258] Signal: Segmentation fault (11) +[b07n13:165258] Signal code: (3) +[b07n13:165258] Failing at address: 0x14170000000000a0 +[b07n13:165258] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n05:169785] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n05:169785] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n05:169785] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n08:121686] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n08:121686] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n08:121686] [ 8] [b06n12:177116] *** Process received signal *** +[b06n12:177116] Signal: Segmentation fault (11) +[b06n12:177116] Signal code: (3) +[b06n12:177116] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n15:137005] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n15:137005] [ 6] [a30n08:163096] *** Process received signal *** +[a30n08:163096] Signal: Segmentation fault (11) +[a30n08:163096] Signal code: (3) +[a30n08:163096] Failing at address: 0x14170000000000a0 +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n11:120239] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n11:120239] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n11:120239] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n15:154383] *** End of error message *** +[a35n02:149230] *** Process received signal *** +[a35n02:149230] Signal: Segmentation fault (11) +[a35n02:149230] Signal code: (3) +[a35n02:149230] Failing at address: 0x14170000000000a0 +[0x1000000504d8] +[b03n11:175593] [ 1] [0x3ffff620ff70] +[b03n11:175593] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n11:175593] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n11:175593] [ 4] [b02n11:02683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n11:02683] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n17:146793] *** End of error message *** +[a35n01:138007] *** Process received signal *** +[a35n01:138007] Signal: Segmentation fault (11) +[a35n01:138007] Signal code: (3) +[a35n01:138007] Failing at address: 0x14170000000000a0 +[a35n01:138007] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n17:126524] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n17:126524] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n17:126524] [ 9] [b04n06:10222] *** Process received signal *** +[b04n06:10222] Signal: Segmentation fault (11) +[b04n06:10222] Signal code: (3) +[b04n06:10222] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n06:154620] [ 4] [0x3fffcf6dbc40] +[b10n03:168933] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n03:168933] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n03:168933] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n15:123038] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n14:163549] [10] [a35n03:135162] [ 0] [0x1000000504d8] +[a35n03:135162] [ 1] [0x3fffed1b0ca0] +[a35n03:135162] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n03:135162] [ 3] [b06n15:161753] [ 0] [0x1000000504d8] +[b06n15:161753] [ 1] [0x3fffd8a0fc60] +[b06n15:161753] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n15:161753] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n12:154344] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n12:154344] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n12:154344] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n12:154344] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n12:154344] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n12:154344] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n12:154344] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n12:154344] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n12:154344] [13] [b07n06:177007] [ 0] [0x1000000504d8] +[b07n06:177007] [ 1] [0x3fffd7329820] +[b07n06:177007] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n06:177007] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n06:177007] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n02:178475] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n02:178475] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n02:178475] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n02:178475] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n02:178475] *** End of error message *** +[b18n03:136665] *** Process received signal *** +[b18n03:136665] Signal: Segmentation fault (11) +[b18n03:136665] Signal code: (3) +[b18n03:136665] Failing at address: 0xb170000000000a0 +[b18n03:136665] [ 0] [0x1000000504d8] +[b18n03:136665] [ 1] [0x3fffedb13490] +[b18n03:136665] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n03:136665] [ 3] [a26n01:65724] *** Process received signal *** +[a26n01:65724] Signal: Segmentation fault (11) +[a26n01:65724] Signal code: (3) +[a26n01:65724] Failing at address: 0xb170000000000a0 +[a26n01:65724] [ 0] [0x1000000504d8] +[a26n01:65724] [ 1] [0x3fffd9877920] +[a26n01:65724] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n01:65724] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n03:125242] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n03:125242] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n03:125242] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n03:125242] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n09:164364] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n04:169706] *** End of error message *** +[b11n05:162569] *** Process received signal *** +[b11n05:162569] Signal: Segmentation fault (11) +[b11n05:162569] Signal code: (3) +[b11n05:162569] Failing at address: 0x14170000000000a0 +[b11n05:162569] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n03:01353] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n03:01353] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n03:01353] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n03:01353] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n03:01353] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n03:01353] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n03:01353] [13] [a35n11:148671] *** Process received signal *** +[a35n11:148671] Signal: Segmentation fault (11) +[a35n11:148671] Signal code: (3) +[a35n11:148671] Failing at address: 0xb170000000000a0 +[a35n11:148671] [ 0] [0x1000000504d8] +[a35n11:148671] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n09:147793] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n09:147793] [ 6] [b15n02:122559] *** Process received signal *** +[b15n02:122559] Signal: Segmentation fault (11) +[b15n02:122559] Signal code: (3) +[b15n02:122559] Failing at address: 0x14170000000000a0 +[b15n02:122559] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n10:161716] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n10:161716] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n17:119464] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n17:119464] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n02:123702] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n02:123702] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n02:123702] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n17:162357] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n17:162357] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n17:162357] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n17:162357] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n06:133955] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n06:133955] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n06:133955] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n06:133955] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n04:13080] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n04:13080] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n04:13080] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n04:13080] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n04:13080] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n04:13080] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n04:13080] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n04:13080] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n04:13080] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n05:135119] [ 5] [a34n01:164960] *** Process received signal *** +[a34n01:164960] Signal: Segmentation fault (11) +[a34n01:164960] Signal code: (3) +[a34n01:164960] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n06:130540] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n06:130540] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n06:130540] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n17:163819] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n17:163819] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n17:163819] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n05:131263] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n05:131263] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n05:131263] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n03:151879] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n03:151879] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n05:154188] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n05:154188] *** End of error message *** +[a28n01:179259] [ 0] [0x1000000504d8] +[a28n01:179259] [ 1] [0x3fffccb24940] +[a28n01:179259] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n01:179259] [ 3] [b21n17:118572] *** Process received signal *** +[b21n17:118572] Signal: Segmentation fault (11) +[b21n17:118572] Signal code: (3) +[b21n17:118572] Failing at address: 0xb170000000000a0 +[b21n17:118572] [ 0] [0x1000000504d8] +[b21n17:118572] [ 1] [0x3fffd434ba20] +[b21n17:118572] [ 2] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n01:171786] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n01:141343] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n01:141343] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n04:142255] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n04:142255] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n08:172429] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n08:172429] [ 6] [a36n03:153852] [ 0] [0x1000000504d8] +[a36n03:153852] [ 1] [0x3fffe26e2420] +[a36n03:153852] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n03:153852] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n03:153852] [ 4] [b21n18:118683] *** Process received signal *** +[b21n18:118683] Signal: Segmentation fault (11) +[b21n18:118683] Signal code: (3) +[b21n18:118683] Failing at address: 0x14170000000000a0 +[a36n02:147882] *** Process received signal *** +[a36n02:147882] Signal: Segmentation fault (11) +[a36n02:147882] Signal code: (3) +[a36n02:147882] Failing at address: 0xb170000000000a0 +[a36n02:147882] [ 0] [0x1000000504d8] +[a36n02:147882] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n16:121930] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n16:121930] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n16:121930] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n16:121930] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n10:164936] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n10:164936] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n10:164936] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n10:164936] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n08:164749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n08:164749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n08:164749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n06:162915] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n06:162915] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n06:162915] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n06:162915] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n11:164993] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n04:04848] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n04:04848] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n04:04848] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n04:04848] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n04:04848] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n03:130349] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n03:130349] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n03:130349] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n03:130349] [14] [0x3ffffb5112e0] +[b01n06:171501] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n06:171501] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n06:171501] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n06:171501] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n05:173079] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n05:173079] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n05:173079] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n05:173079] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n05:173079] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n05:173079] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n05:173079] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n05:173079] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n05:173079] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n05:173079] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n05:173079] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n05:173079] *** End of error message *** +[0x1000000504d8] +[b14n02:139026] [ 1] [0x3fffe6230a70] +[b14n02:139026] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n02:139026] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n02:139026] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n02:139026] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n02:139026] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n02:139026] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n02:139026] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n02:139026] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n02:139026] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n06:163674] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n06:163674] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n06:163674] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n06:163674] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n01:128002] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n06:120732] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n06:120732] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n06:120732] [ 8] [b15n05:120330] *** Process received signal *** +[b15n05:120330] Signal: Segmentation fault (11) +[b15n05:120330] Signal code: (3) +[b15n05:120330] Failing at address: 0x14170000000000a0 +[b15n05:120330] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n11:160795] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n11:160795] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n12:138734] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n12:138734] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n12:138734] [ 7] [0x1000000504d8] +[b19n17:123237] [ 1] [0x3fffeff20d60] +[b19n17:123237] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n17:123237] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n17:123237] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n17:123237] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n04:126858] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n04:126858] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n04:126858] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n04:126858] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n04:126858] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n16:164550] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n16:164550] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n05:126778] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n05:126778] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n05:126778] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n05:126778] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n05:126778] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n05:126778] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n05:126778] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n07:124348] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n07:124348] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n07:124348] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n01:165123] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n09:155759] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n09:155759] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n13:153675] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n13:153675] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n13:153675] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n13:153675] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n13:153675] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n06:157788] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n06:157788] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n06:157788] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n06:157788] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n06:157788] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n06:157788] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n13:10195] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n13:10195] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n11:117473] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n12:178600] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n12:178600] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n12:178600] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n12:178600] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n12:178600] [14] [b07n10:16971] *** Process received signal *** +[b07n10:16971] Signal: Segmentation fault (11) +[b07n10:16971] Signal code: (3) +[b07n10:16971] Failing at address: 0x14170000000000a0 +[b07n10:16971] [ 0] [b01n11:03982] *** Process received signal *** +[b01n11:03982] Signal: Segmentation fault (11) +[b01n11:03982] Signal code: (3) +[b01n11:03982] Failing at address: 0x14170000000000a0 +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n06:161724] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n11:120583] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n11:120583] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n11:120583] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n11:120583] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n06:117528] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n06:117528] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n07:160801] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n07:160801] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n07:160801] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n17:176412] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n17:176412] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n17:176412] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n17:176412] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n18:179169] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n13:123514] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n13:123514] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n13:123514] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n04:158631] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n04:158631] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n04:158631] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n14:178326] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n14:178326] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n14:178326] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n14:178326] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n17:168154] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n17:168154] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n17:168154] *** End of error message *** +[0x1000000504d8] +[b11n03:154292] [ 1] [0x3fffc371c350] +[b11n03:154292] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n03:154292] [ 3] [0x3fffd3baae20] +[b23n09:119631] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n09:119631] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n09:119631] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n09:119631] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n11:134517] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n11:134517] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n11:134517] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n11:134517] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n11:134517] *** End of error message *** +[b22n04:44759] Signal: Segmentation fault (11) +[b22n04:44759] Signal code: (3) +[b22n04:44759] Failing at address: 0x14170000000000a0 +[b22n04:44759] [ 0] [0x1000000504d8] +[b22n04:44759] [ 1] [0x3fffd3f5c670] +[b22n04:44759] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n04:44759] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n13:171106] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n13:171106] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n13:171106] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n04:145748] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n04:145748] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n04:145748] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n05:139437] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n05:139437] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n05:139437] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n10:11491] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n10:11491] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n10:11491] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n10:11491] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n10:11491] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n07:168960] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n07:168960] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n07:168960] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n07:168960] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n03:06823] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n12:123562] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n01:163390] [ 9] [b22n10:118389] Signal: Segmentation fault (11) +[b22n10:118389] Signal code: (3) +[b22n10:118389] Failing at address: 0x14170000000000a0 +[b22n10:118389] [ 0] [0x1000000504d8] +[b22n10:118389] [ 1] [0x3fffc7103160] +[b22n10:118389] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n10:118389] [ 3] [a29n05:174778] *** Process received signal *** +[a29n05:174778] Signal: Segmentation fault (11) +[a29n05:174778] Signal code: (3) +[a29n05:174778] Failing at address: 0xb170000000000a0 +[a29n05:174778] [ 0] [0x1000000504d8] +[a29n05:174778] [ 1] [0x3fffc5caa5d0] +[a29n05:174778] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n05:174778] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n10:169209] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n05:139912] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n05:139912] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n13:123051] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n13:123051] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n13:123051] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n13:123051] [10] [b04n01:96753] *** Process received signal *** +[b04n01:96753] Signal: Segmentation fault (11) +[b04n01:96753] Signal code: (3) +[b04n01:96753] Failing at address: 0x5170000000000a0 +[b04n01:96753] [ 0] [a30n13:154234] *** Process received signal *** +[a30n13:154234] Signal: Segmentation fault (11) +[a30n13:154234] Signal code: (3) +[a30n13:154234] Failing at address: 0x14170000000000a0 +[a30n13:154234] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n12:160963] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n12:160963] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n12:160963] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n12:160963] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n12:160963] [13] [ 1] [0x3fffdf93cab0] +[a34n07:165087] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n07:165087] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n07:165087] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n03:172649] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n03:172649] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n03:172649] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n03:172649] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n13:133626] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n13:133626] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n13:133626] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n11:130803] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n11:130803] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n11:130803] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n11:130803] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n11:130803] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n11:130803] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n08:176682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n08:176682] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n08:176682] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n01:170767] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n15:120129] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n15:120129] [ 6] [b11n16:154181] *** Process received signal *** +[b11n16:154181] Signal: Segmentation fault (11) +[b11n16:154181] Signal code: (3) +[b11n16:154181] Failing at address: 0x14170000000000a0 +[b11n16:154181] [ 0] [0x1000000504d8] +[b11n16:154181] [ 1] [0x3fffd2e6f580] +[b11n16:154181] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n16:154181] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n06:179454] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n06:179454] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n18:128543] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n18:128543] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n18:128543] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n18:128543] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n18:128543] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n10:140413] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n10:140413] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n10:140413] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n10:140413] [ 9] [0x1000000504d8] +[a33n11:156152] [ 1] [0x3fffe35c5c50] +[a33n11:156152] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n11:156152] [ 3] [a32n14:152745] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n14:152745] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n14:152745] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n05:120227] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n05:120227] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n05:120227] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n05:120227] [ 8] [0x1000000504d8] +[b20n02:122177] [ 1] [0x3fffd0b11480] +[b20n02:122177] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n02:122177] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n02:122177] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n15:139749] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n10:10925] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n10:10925] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n10:10925] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n10:155400] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n10:155400] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n02:74430] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n10:169681] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n10:169681] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n10:169681] *** End of error message *** +[a02n10:135853] *** Process received signal *** +[a02n10:135853] Signal: Segmentation fault (11) +[a02n10:135853] Signal code: (3) +[a02n10:135853] Failing at address: 0x14170000000000a0 +[a02n10:135853] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n08:127442] [ 5] [b18n16:124957] *** Process received signal *** +[b18n16:124957] Signal: Segmentation fault (11) +[b18n16:124957] Signal code: (3) +[b18n16:124957] Failing at address: 0xb170000000000a0 +[b18n16:124957] [ 0] [a30n18:10723] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n03:164495] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n03:164495] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n03:164495] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n03:164495] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n03:164495] *** End of error message *** +[0x3fffcc854670] +[b19n01:121746] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n01:121746] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n01:121746] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n01:121746] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n01:121746] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n01:121746] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n01:121746] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n01:121746] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n01:121746] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n01:121746] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n01:121746] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n01:121746] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n01:121746] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n07:156489] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n10:120978] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n12:160191] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n12:160191] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n12:160191] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n12:160191] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n12:160191] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n13:155847] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n13:155847] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n13:155847] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n13:155847] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n13:155847] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n13:155847] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n13:155847] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n13:155847] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n13:155847] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n13:155847] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n13:155847] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n13:155847] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n04:130745] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n04:130745] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n04:130745] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n12:122873] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n12:122873] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n02:125374] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n02:125374] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n02:125374] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n09:170656] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n09:170656] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n09:170656] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n07:45308] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n05:176909] [ 5] [0x3fffd0ca07b0] +[a02n08:146117] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n08:146117] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n08:146117] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n08:146117] [ 5] [b15n10:122760] *** Process received signal *** +[b15n10:122760] Signal: Segmentation fault (11) +[b15n10:122760] Signal code: (3) +[b15n10:122760] Failing at address: 0xb170000000000a0 +[b15n10:122760] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n07:172880] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n07:172880] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n07:172880] [ 6] [b17n03:135035] *** Process received signal *** +[b17n03:135035] Signal: Segmentation fault (11) +[b17n03:135035] Signal code: (3) +[b17n03:135035] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n07:02839] [ 5] [b17n14:126122] *** Process received signal *** +[b17n14:126122] Signal: Segmentation fault (11) +[b17n14:126122] Signal code: (3) +[b17n14:126122] Failing at address: 0x14170000000000a0 +[b17n14:126122] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n03:147533] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n03:147533] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n03:147533] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n03:147533] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n03:147533] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n03:147533] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n11:134745] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n11:134745] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n11:134745] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n04:136994] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n04:136994] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n04:136994] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n04:136994] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n08:118551] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n08:118551] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n08:118551] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n08:118551] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n08:118551] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n08:118551] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n08:118551] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n08:118551] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n05:05605] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n05:05605] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n03:68828] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n03:68828] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n03:68828] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n03:68828] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n03:68828] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n05:173819] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n05:173819] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n05:173819] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n05:173819] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n05:173819] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n05:173819] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n05:173819] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n05:173819] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n01:123569] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n01:123569] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n01:123569] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n01:123569] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n01:123569] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n12:145265] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n12:145265] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n12:145265] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n12:145265] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n03:119831] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n03:119831] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n14:139843] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n14:139843] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n14:139843] *** End of error message *** +[0x3fffdfe6e650] +[a32n08:165284] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n08:165284] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n08:165284] [ 4] [b18n07:125447] [ 0] [0x1000000504d8] +[b18n07:125447] [ 1] [0x3fffca1556d0] +[b18n07:125447] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n07:125447] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n03:124681] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n13:137748] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n13:137748] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n13:137748] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n13:137748] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n13:137748] *** End of error message *** +[b18n11:128956] [ 0] [b14n14:122440] Signal: Segmentation fault (11) +[b14n14:122440] Signal code: (3) +[b14n14:122440] Failing at address: 0x14170000000000a0 +[b14n14:122440] [ 0] [0x1000000504d8] +[b14n14:122440] [ 1] [0x3fffed46ef30] +[b14n14:122440] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n14:122440] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n07:163252] *** End of error message *** +[b09n04:177594] [ 0] [0x1000000504d8] +[b09n04:177594] [ 1] [0x3fffd5dea160] +[b09n04:177594] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n04:177594] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n04:177594] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n09:143896] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n14:177222] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n14:177222] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n14:177222] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n14:177222] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n14:177222] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n18:121307] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n18:121307] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n18:121307] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n18:121307] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n08:124115] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n08:124115] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n08:124115] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n08:124115] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n08:124115] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n04:117351] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n04:117351] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n04:117351] [ 8] [b11n07:156143] *** Process received signal *** +[b11n07:156143] Signal: Segmentation fault (11) +[b11n07:156143] Signal code: (3) +[b11n07:156143] Failing at address: 0xb170000000000a0 +[b11n07:156143] [ 0] [0x1000000504d8] +[b11n07:156143] [ 1] [0x3ffff18d32c0] +[b11n07:156143] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n07:156143] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n07:156143] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n07:156143] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n07:156143] [ 6] [b13n08:144851] *** Process received signal *** +[b13n08:144851] Signal: Segmentation fault (11) +[b13n08:144851] Signal code: (3) +[b13n08:144851] Failing at address: 0x14170000000000a0 +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n07:149476] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n18:122552] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n18:122552] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n18:122552] [ 6] [a35n15:148164] [ 0] [0x1000000504d8] +[a35n15:148164] [ 1] [0x3ffff47ebcc0] +[a35n15:148164] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n15:148164] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n16:130377] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n16:130377] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n10:134733] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n10:134733] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n10:134733] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n10:134733] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n10:134733] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n05:121822] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n05:121822] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n05:121822] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n05:121822] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n16:147284] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n15:32399] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n15:32399] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n15:32399] [ 6] [a32n12:164379] *** Process received signal *** +[a32n12:164379] Signal: Segmentation fault (11) +[a32n12:164379] Signal code: (3) +[a32n12:164379] Failing at address: 0xb170000000000a0 +[a32n12:164379] [ 0] [b09n01:163021] *** Process received signal *** +[b09n01:163021] Signal: Segmentation fault (11) +[b09n01:163021] Signal code: (3) +[b09n01:163021] Failing at address: 0xb170000000000a0 +[b09n01:163021] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n14:136136] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n14:136136] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n14:136136] [13] [0x1000000504d8] +[b02n04:168488] [ 1] [0x3fffc17aabe0] +[b02n04:168488] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n04:168488] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n04:168488] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n04:168488] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n11:163260] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n11:163260] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n15:151248] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n15:151248] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n17:125805] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n12:178716] [ 4] [b22n15:120060] *** Process received signal *** +[b22n15:120060] Signal: Segmentation fault (11) +[b22n15:120060] Signal code: (3) +[b22n15:120060] Failing at address: 0x14170000000000a0 +[b22n15:120060] [ 0] [0x1000000504d8] +[b22n15:120060] [ 1] [0x3ffff687ee90] +[b22n15:120060] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n15:120060] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n13:117508] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n13:117508] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n13:117508] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n13:117508] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n13:138087] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n17:168162] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n17:168162] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n14:150130] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n14:150130] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n14:119486] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n14:119486] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n14:119486] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n14:119486] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n04:126348] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n04:126348] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n04:126348] [ 8] [0x1000000504d8] +[b06n13:166922] [ 1] [0x3fffe46f3900] +[b06n13:166922] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n13:166922] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n13:166922] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n02:119531] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n02:119531] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n02:119531] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n11:124432] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n13:168158] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n13:168158] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n13:168158] [ 9] [a31n03:158659] [ 0] [0x1000000504d8] +[a31n03:158659] [ 1] [0x3fffcafa23b0] +[a31n03:158659] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n03:158659] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n03:158659] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n10:162607] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n09:161228] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n09:161228] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n09:161228] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n09:161228] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n09:161228] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n09:161228] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n09:161228] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n09:161228] [13] [a29n11:162473] *** Process received signal *** +[a29n11:162473] Signal: Segmentation fault (11) +[a29n11:162473] Signal code: (3) +[a29n11:162473] Failing at address: 0x14170000000000a0 +[a29n11:162473] [ 0] [0x1000000504d8] +[a29n11:162473] [ 1] [0x3fffdff53350] +[a29n11:162473] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n11:162473] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n11:162473] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n16:173547] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n16:173547] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n18:164799] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n18:164799] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n15:124802] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n15:124802] *** End of error message *** +[b13n16:128442] [ 0] [0x1000000504d8] +[b13n16:128442] [ 1] [0x3fffc4879e10] +[b13n16:128442] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n16:128442] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n16:128442] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n18:08431] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n18:08431] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n18:08431] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n18:08431] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n13:123884] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n13:123884] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n13:123884] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n13:123884] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n13:123884] [10] [0x1000000504d8] +[b02n05:164942] [ 1] [0x3fffd8193bc0] +[b02n05:164942] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n05:164942] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n05:164942] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n05:164942] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n12:120127] [ 5] [a27n08:54623] [ 0] [0x1000000504d8] +[a27n08:54623] [ 1] [0x3fffe6332920] +[a27n08:54623] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n08:54623] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n08:54623] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n18:131584] *** End of error message *** +[a31n11:153419] *** Process received signal *** +[a31n11:153419] Signal: Segmentation fault (11) +[a31n11:153419] Signal code: (3) +[a31n11:153419] Failing at address: 0x14170000000000a0 +[a31n11:153419] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n12:132102] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n02:167599] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n02:167599] *** End of error message *** +[ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n04:166205] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n04:166205] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n14:133063] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n08:131380] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n08:131380] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n08:116174] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n08:116174] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n08:116174] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n08:116174] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n08:116174] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n08:116174] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n16:165925] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n16:165925] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n16:165925] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n17:124164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n17:124164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n17:124164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n17:124164] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n07:117358] [10] [b14n06:127272] [ 0] [0x1000000504d8] +[b14n06:127272] [ 1] [0x3fffc2dd09f0] +[b14n06:127272] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n06:127272] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n06:127272] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n06:127272] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n06:127272] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n07:133376] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n06:167370] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n06:167370] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n06:167370] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n06:167370] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n06:167370] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n06:167370] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n02:176048] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n02:176048] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n02:176048] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n02:176048] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n02:176048] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n02:176048] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n17:129929] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n17:129929] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n17:129929] [13] [0x1000000504d8] +[a02n03:38499] [ 1] [0x3fffefc7da30] +[a02n03:38499] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n03:38499] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n03:38499] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n03:38499] [ 5] [b19n03:123618] *** Process received signal *** +[b19n03:123618] Signal: Segmentation fault (11) +[b19n03:123618] Signal code: (3) +[b19n03:123618] Failing at address: 0x14170000000000a0 +[b19n03:123618] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n11:170895] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n11:170895] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n11:170895] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n11:170895] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n11:170895] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n11:170895] *** End of error message *** +[0x3fffdf46d9c0] +[b10n11:160349] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n11:160349] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n11:160349] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n11:160349] [ 5] [a29n16:157887] *** Process received signal *** +[a29n16:157887] Signal: Segmentation fault (11) +[a29n16:157887] Signal code: (3) +[a29n16:157887] Failing at address: 0xb170000000000a0 +[a29n16:157887] [ 0] [0x1000000504d8] +[a29n16:157887] [ 1] [0x3fffeb673fd0] +[a29n16:157887] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n16:157887] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n13:114395] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n13:114395] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n13:114395] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n13:114395] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n16:116523] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n16:116523] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n16:116523] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n16:116523] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n16:116523] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n10:142932] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n10:142932] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n10:142932] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n01:128831] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n01:128831] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n01:128831] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n01:128831] [13] [a02n04:41669] *** Process received signal *** +[a02n04:41669] Signal: Segmentation fault (11) +[a02n04:41669] Signal code: (3) +[a02n04:41669] Failing at address: 0xb170000000000a0 +[a02n04:41669] [ 0] [0x1000000504d8] +[a02n04:41669] [ 1] [0x3fffe6536cf0] +[a02n04:41669] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n04:41669] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n04:41669] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n16:127350] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n16:127350] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n07:180191] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n14:176069] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n18:179697] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n18:179697] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n18:179697] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n18:179697] [ 9] [b19n09:123416] *** Process received signal *** +[b19n09:123416] Signal: Segmentation fault (11) +[b19n09:123416] Signal code: (3) +[b19n09:123416] Failing at address: 0xb170000000000a0 +[b19n09:123416] [ 0] [0x1000000504d8] +[b19n09:123416] [ 1] [0x3ffff919cac0] +[b19n09:123416] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n09:123416] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n13:122030] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n01:48802] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n01:48802] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n01:48802] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n01:48802] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n01:48802] [11] [0x3ffff4e5f4b0] +[b03n12:167674] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n12:167674] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n12:167674] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n12:167674] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n02:06562] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n02:06562] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n09:51333] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n09:51333] *** End of error message *** +[b08n12:12763] *** Process received signal *** +[b08n12:12763] Signal: Segmentation fault (11) +[b08n12:12763] Signal code: (3) +[b08n12:12763] Failing at address: 0x14170000000000a0 +[b08n12:12763] [ 0] [0x1000000504d8] +[b08n12:12763] [ 1] [0x3fffcd6d9c20] +[b08n12:12763] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n12:12763] [ 3] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n17:149906] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n15:177054] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n15:177054] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n15:177054] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n15:177054] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n15:177054] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n15:177054] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n12:122967] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n12:122967] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n14:128867] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n14:128867] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n14:128867] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n17:03843] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b16n14:133636] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b16n14:133636] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b16n14:133636] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b16n14:133636] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n06:126379] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n06:126379] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n17:15870] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n17:15870] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n17:15870] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n17:15870] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n17:15870] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n05:135162] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n05:135162] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n05:135162] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n14:169521] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n14:169521] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n14:169521] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n14:169521] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n14:136659] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n14:136659] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n14:136659] *** End of error message *** +[b05n08:08953] Signal: Segmentation fault (11) +[b05n08:08953] Signal code: (3) +[b05n08:08953] Failing at address: 0x14170000000000a0 +[b05n08:08953] [ 0] [0x1000000504d8] +[b05n08:08953] [ 1] [0x1000000504d8] +[b17n18:126906] [ 1] [0x3fffcd1eed40] +[b17n18:126906] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n18:126906] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n18:126906] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n18:126906] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n06:143008] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n06:143008] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n06:143008] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n06:143008] [10] [0x3fffdf45b9d0] +[a33n01:141815] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n01:141815] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n01:141815] [ 4] [b02n01:167300] *** Process received signal *** +[b02n01:167300] Signal: Segmentation fault (11) +[b02n01:167300] Signal code: (3) +[b02n01:167300] Failing at address: 0x14170000000000a0 +[b02n01:167300] [ 0] [0x1000000504d8] +[b02n01:167300] [ 1] [0x3ffff3bcb240] +[b02n01:167300] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n01:167300] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n13:176364] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n06:01840] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n06:01840] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n06:01840] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n06:01840] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n06:01840] [14] [a28n10:176306] [ 0] [0x1000000504d8] +[a28n10:176306] [ 1] [0x3fffc7d6af40] +[a28n10:176306] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n10:176306] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n10:176306] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n10:176306] [ 5] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n07:138205] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n07:138205] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n07:138205] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n07:138205] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n08:147820] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n08:147820] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n08:147820] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n08:147820] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n08:147820] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n08:147820] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n08:147820] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n07:132087] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n07:132087] [ 5] [a33n04:169909] *** Process received signal *** +[a33n04:169909] Signal: Segmentation fault (11) +[a33n04:169909] Signal code: (3) +[a33n04:169909] Failing at address: 0xb170000000000a0 +[a33n04:169909] [ 0] [0x1000000504d8] +[a33n04:169909] [ 1] [a36n17:150930] *** Process received signal *** +[a36n17:150930] Signal: Segmentation fault (11) +[a36n17:150930] Signal code: (3) +[a36n17:150930] Failing at address: 0x14170000000000a0 +[a36n17:150930] [ 0] [0x1000000504d8] +[a36n17:150930] [ 1] [0x3fffdb3666e0] +[a36n17:150930] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n17:150930] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n02:164492] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n02:164492] [ 5] [b19n07:131669] *** Process received signal *** +[b19n07:131669] Signal: Segmentation fault (11) +[b19n07:131669] Signal code: (3) +[b19n07:131669] Failing at address: 0xb170000000000a0 +[b19n07:131669] [ 0] [0x1000000504d8] +[b19n07:131669] [ 1] [0x3fffc749fc30] +[b19n07:131669] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n07:131669] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n11:137869] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n11:137869] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n11:137869] [ 6] [b02n02:166743] [ 0] [0x1000000504d8] +[b02n02:166743] [ 1] [0x3fffe8a0a250] +[b02n02:166743] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n02:166743] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n02:166743] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n02:166743] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n02:166743] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n02:166743] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n02:166743] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n16:01568] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n16:01568] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n01:128154] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n01:128154] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n03:10288] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n03:10288] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n03:10288] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n03:10288] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n09:121752] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n09:121752] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n09:121752] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n09:121752] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n13:123706] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n13:123706] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n13:123706] [ 5] [b11n11:152985] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n11:01172] *** End of error message *** +[0x3fffeeff8030] +[a27n18:28220] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a27n18:28220] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n18:28220] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n18:28220] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n18:28220] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n18:28220] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n18:28220] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n18:28220] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n18:28220] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n18:28220] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n18:28220] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n18:28220] [13] [b09n11:171962] [ 0] [0x1000000504d8] +[b09n11:171962] [ 1] [0x3fffef31ffd0] +[b09n11:171962] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n11:171962] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n11:171962] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n11:171962] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n11:171962] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n01:130408] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n01:130408] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n01:130408] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n01:130408] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n14:161619] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n08:124722] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n08:124722] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n08:124722] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n09:173752] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n09:173752] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n09:173752] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n09:173752] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n16:122492] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n16:122492] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n15:85942] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n15:85942] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n15:85942] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n10:130580] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n16:153292] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n16:153292] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n16:153292] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n16:153292] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n02:143965] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n12:148378] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n01:173782] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n01:173782] *** End of error message *** +[b05n12:11142] [ 0] [0x1000000504d8] +[b05n12:11142] [ 1] [0x3fffe3bf9cb0] +[b05n12:11142] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n12:11142] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n12:11142] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n09:119545] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n09:119545] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n09:119545] *** End of error message *** +[0x3ffff84c4a70] +[a31n12:156915] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n12:156915] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n12:156915] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n12:156915] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n12:156915] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n12:156915] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n12:156915] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n02:120188] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n02:120188] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n02:120188] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n02:120188] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n03:127629] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n17:119580] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n17:119580] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n17:119580] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n17:119580] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n04:175997] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n04:175997] [ 6] [0x1000000504d8] +[b18n09:123810] [ 1] [0x3fffe1e88550] +[b18n09:123810] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n09:123810] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n15:148582] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n15:148582] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n01:145731] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n01:145731] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n02:163318] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n02:163318] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n07:125470] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n07:125470] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n07:125470] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n07:125470] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n07:125470] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n07:125470] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n07:125470] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n08:163164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n08:163164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n08:163164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n08:163164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n08:163164] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n04:138156] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n04:138156] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n04:138156] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n04:138156] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n04:138156] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n04:138156] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n04:138156] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n04:138156] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n04:138156] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n04:138156] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n04:138156] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n17:164562] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n17:164562] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n17:164562] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n17:164562] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n17:164562] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n10:122205] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n10:122205] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n10:122205] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n10:122205] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n10:122205] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n10:122205] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n10:122205] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n10:122205] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n10:122205] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n10:122205] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n11:147713] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n11:147713] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n11:147713] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n07:127357] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n07:127357] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n12:115640] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n12:115640] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n12:115640] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n12:115640] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n12:115640] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n08:156369] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n08:156369] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n08:156369] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n08:156369] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n08:156369] [a32n13:163191] *** Process received signal *** +[a32n13:163191] Signal: Segmentation fault (11) +[a32n13:163191] Signal code: (3) +[a32n13:163191] Failing at address: 0xb170000000000a0 +[a32n13:163191] [ 0] [0x1000000504d8] +[a32n13:163191] [ 1] [0x3fffd8e3d0f0] +[a32n13:163191] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n13:163191] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n13:163191] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n13:163191] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n13:163191] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n13:163191] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n13:163191] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n13:163191] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n13:163191] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n13:163191] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n13:163191] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n13:163191] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n13:163191] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n13:163191] *** End of error message *** +[0x1000000504d8] +[b07n13:165259] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n05:169785] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n08:121686] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n08:121686] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n08:121686] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n15:137004] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n15:137004] [ 6] [b06n12:177116] [ 0] [0x1000000504d8] +[b06n12:177116] [ 1] [0x3fffee6c8000] +[b06n12:177116] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n12:177116] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n12:177116] [ 4] [a30n08:163097] *** Process received signal *** +[a30n08:163097] Signal: Segmentation fault (11) +[a30n08:163097] Signal code: (3) +[a30n08:163097] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n11:120238] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n11:120238] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n11:120238] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n11:120238] [ 9] [a35n02:149230] [ 0] [0x1000000504d8] +[a35n02:149230] [ 1] [0x3fffff507300] +[a35n02:149230] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n02:149230] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n15:154382] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n15:154382] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n15:154382] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n15:154382] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n15:154382] *** End of error message *** +[b03n11:175594] *** Process received signal *** +[b03n11:175594] Signal: Segmentation fault (11) +[b03n11:175594] Signal code: (3) +[b03n11:175594] Failing at address: 0x14170000000000a0 +[b03n11:175594] [ 0] [0x1000000504d8] +[b03n11:175594] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n11:02682] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n11:02682] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n11:02682] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n11:02682] [ 9] [a35n17:146796] *** Process received signal *** +[a35n17:146796] Signal: Segmentation fault (11) +[a35n17:146796] Signal code: (3) +[a35n17:146796] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n17:126524] [10] [a35n01:138006] Signal: Segmentation fault (11) +[a35n01:138006] Signal code: (3) +[a35n01:138006] Failing at address: 0x14170000000000a0 +[a35n01:138006] [ 0] [0x1000000504d8] +[a35n01:138006] [ 1] [0x3ffffc30b0f0] +[a35n01:138006] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n01:138006] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n03:168933] [ 5] [b04n06:10223] *** Process received signal *** +[b04n06:10223] Signal: Segmentation fault (11) +[b04n06:10223] Signal code: (3) +[b04n06:10223] Failing at address: 0x14170000000000a0 +[b04n06:10223] [ 0] [0x1000000504d8] +[b04n06:10223] [ 1] [0x3ffff1b24300] +[b04n06:10223] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n15:123037] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n15:123037] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n15:123037] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n15:123037] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n15:123037] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n15:123037] [13] [a31n09:164364] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n14:163549] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n03:135161] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n03:135161] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n03:135161] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n03:135161] [12] [b07n06:177006] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n15:161753] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n12:154345] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n12:154345] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n12:154345] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n12:154345] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n12:154345] *** End of error message *** +[a28n02:178477] *** Process received signal *** +[a28n02:178477] Signal: Segmentation fault (11) +[a28n02:178477] Signal code: (3) +[a28n02:178477] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n03:136665] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n03:136665] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n03:136665] [ 6] [a26n01:65725] *** Process received signal *** +[a26n01:65725] Signal: Segmentation fault (11) +[a26n01:65725] Signal code: (3) +[a26n01:65725] Failing at address: 0x14170000000000a0 +[a26n01:65725] [ 0] [0x1000000504d8] +[a26n01:65725] [ 1] [0x3fffd7439070] +[a26n01:65725] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a26n01:65725] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n03:125243] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n03:125243] [ 5] [a32n06:154619] *** Process received signal *** +[a32n06:154619] Signal: Segmentation fault (11) +[a32n06:154619] Signal code: (3) +[a32n06:154619] Failing at address: 0xb170000000000a0 +[a32n06:154619] [ 0] [0x1000000504d8] +[a32n06:154619] [ 1] [0x3ffff878e4a0] +[a32n06:154619] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n06:154619] [ 3] [b11n05:162568] [ 0] [0x1000000504d8] +[b11n05:162568] [ 1] [0x3fffe6ae7150] +[b11n05:162568] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n05:162568] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n03:01354] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n03:01354] *** End of error message *** +[0x3fffcd45cb90] +[a35n11:148672] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n11:148672] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n11:148672] [b01n04:169703] *** Process received signal *** +[b01n04:169703] Signal: Segmentation fault (11) +[b01n04:169703] Signal code: (3) +[b01n04:169703] Failing at address: 0xb170000000000a0 +[0x1000000504d8] +[b15n02:122559] [ 1] [0x3fffc6873630] +[b15n02:122559] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n02:122559] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n02:122559] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n09:147794] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n09:147794] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n09:147794] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n17:119465] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n02:123702] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n02:123702] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n02:123702] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n06:133955] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n06:133955] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n05:135119] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n05:135119] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n05:135119] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n01:164961] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n01:164961] [ 5] [a32n03:151882] *** Process received signal *** +[a32n03:151882] Signal: Segmentation fault (11) +[a32n03:151882] Signal code: (3) +[a32n03:151882] Failing at address: 0x5170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n06:130541] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n06:130541] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n06:130541] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n06:130541] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n17:163816] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n05:131263] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n05:131263] [11] [0x1000000504d8] +[a28n01:179260] [ 1] [0x3fffdebd0ce0] +[a28n01:179260] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n01:179260] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n01:179260] [ 4] [0x1000000504d8] +[b21n17:118571] [ 1] [0x3fffe82dc7a0] +[b21n17:118571] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n17:118571] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n17:118571] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n01:141344] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n01:141344] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n01:141344] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n02:147881] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n02:147881] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n02:147881] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n02:147881] [ 8] [a36n03:153853] *** Process received signal *** +[a36n03:153853] Signal: Segmentation fault (11) +[a36n03:153853] Signal code: (3) +[a36n03:153853] Failing at address: 0x5170000000000a0 +[a36n03:153853] [ 0] [0x1000000504d8] +[a36n03:153853] [ 1] [0x3ffffb245320] +[a36n03:153853] [ 2] [b21n18:118684] *** Process received signal *** +[b21n18:118684] Signal: Segmentation fault (11) +[b21n18:118684] Signal code: (3) +[b21n18:118684] Failing at address: 0x14170000000000a0 +[b21n18:118684] [ 0] [0x1000000504d8] +[b21n18:118684] [ 1] [0x3fffc79222a0] +[b21n18:118684] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n18:118684] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n08:172431] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n08:172431] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n08:172431] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n08:172431] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n08:172431] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n16:121929] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n10:164937] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n10:164937] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n10:164937] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n10:164937] *** End of error message *** +[b08n08:164748] *** Process received signal *** +[b08n08:164748] Signal: Segmentation fault (11) +[b08n08:164748] Signal code: (3) +[b08n08:164748] Failing at address: 0xb170000000000a0 +[b08n08:164748] [ 0] [0x1000000504d8] +[b08n08:164748] [ 1] [0x3fffd683c460] +[b08n08:164748] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n08:164748] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n08:164748] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n08:164748] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n08:164748] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n08:164748] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n08:164748] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n06:162915] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n06:162915] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n11:164994] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n03:130349] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n04:04848] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n04:04848] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n06:171500] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n06:171500] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n02:139027] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n02:139027] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n02:139027] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n02:139027] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n02:139027] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n02:139027] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n02:139027] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n06:163675] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n06:163675] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n06:163675] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n01:128002] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n06:120732] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n06:120732] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n12:138735] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n12:138735] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n12:138735] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n12:138735] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n12:138735] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n12:138735] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n12:138735] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n05:120329] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n05:120329] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n11:160796] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n11:160796] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n11:160796] [ 6] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n17:123236] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n17:123236] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n17:123236] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n17:123236] [ 6] [b10n16:164552] *** Process received signal *** +[b10n16:164552] Signal: Segmentation fault (11) +[b10n16:164552] Signal code: (3) +[b10n16:164552] Failing at address: 0xb170000000000a0 +[b10n16:164552] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n04:126857] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n04:126857] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n04:126857] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n04:126857] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n04:126857] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n05:126779] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n05:126779] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n05:126779] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n05:126779] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n05:126779] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n07:124349] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n07:124349] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n01:165122] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n01:165122] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n01:165122] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n01:165122] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n01:165122] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n13:153676] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n13:153676] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n09:155760] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n09:155760] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n09:155760] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n09:155760] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n09:155760] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n09:155760] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n06:157787] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n06:157787] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n06:157787] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n06:157787] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n06:157787] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n06:157787] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n06:157787] *** End of error message *** +[0x1000000504d8] +[b07n10:16971] [ 1] [0x3fffe9c6c1b0] +[b07n10:16971] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n10:16971] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n10:16971] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n10:16971] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n11:117472] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n11:117472] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n11:117472] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n11:117472] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n11:117472] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n11:117472] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n11:117472] *** End of error message *** +[b01n12:178602] *** Process received signal *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n13:10196] *** End of error message *** +[b01n11:03983] *** Process received signal *** +[b01n11:03983] Signal: Segmentation fault (11) +[b01n11:03983] Signal code: (3) +[b01n11:03983] Failing at address: 0xb170000000000a0 +[b01n11:03983] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n06:161724] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n11:120584] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n11:120584] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n11:120584] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n06:117528] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n06:117528] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n06:117528] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n06:117528] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n07:160800] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n07:160800] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n07:160800] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n07:160800] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n07:160800] *** End of error message *** +[a28n18:179167] *** Process received signal *** +[a28n18:179167] Signal: Segmentation fault (11) +[a28n18:179167] Signal code: (3) +[a28n18:179167] Failing at address: 0xb170000000000a0 +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n13:123514] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n17:176412] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n04:158632] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n04:158632] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n04:158632] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n04:158632] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n04:158632] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n04:158632] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n11:134519] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n11:134519] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n14:178325] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n17:168155] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n03:154293] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n03:154293] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n09:119632] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n09:119632] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n09:119632] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n09:119632] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n09:119632] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n09:119632] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n04:44758] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n04:44758] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n04:44758] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n04:44758] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n04:44758] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n04:44758] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n04:145748] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n13:171107] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n13:171107] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n13:171107] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n13:171107] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n13:171107] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n05:139437] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n05:139437] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n07:168961] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n10:11491] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n10:11491] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n01:163390] [10] [0x1000000504d8] +[a29n05:174779] [ 1] [0x3fffdd15a5d0] +[a29n05:174779] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n05:174779] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n05:174779] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n10:118389] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n12:123561] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n12:123561] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n12:123561] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n12:123561] [10] [b03n03:06822] *** Process received signal *** +[b03n03:06822] Signal: Segmentation fault (11) +[b03n03:06822] Signal code: (3) +[b03n03:06822] Failing at address: 0x14170000000000a0 +[b03n03:06822] [ 0] [0x1000000504d8] +[b04n01:96753] [ 1] [0x3fffe9107ac0] +[b04n01:96753] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n01:96753] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n01:96753] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n10:169210] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n13:123052] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n13:123052] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n13:123052] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n13:123052] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n13:123052] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n05:139913] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n05:139913] *** End of error message *** +[a30n13:154233] [ 0] [0x1000000504d8] +[a30n13:154233] [ 1] [0x3fffee89f9f0] +[a30n13:154233] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n13:154233] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n13:154233] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n13:154233] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n13:154233] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n12:160962] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n12:160962] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n12:160962] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n05:120228] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n05:120228] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n05:120228] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n05:120228] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n03:172650] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n03:172650] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n03:172650] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n03:172650] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n03:172650] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n13:133627] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n13:133627] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n13:133627] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n01:170768] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n01:170768] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n01:170768] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n01:170768] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n01:170768] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n11:130804] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n11:130804] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n08:176682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n08:176682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n08:176682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n08:176682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n08:176682] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n16:154181] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n16:154181] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n15:120130] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n15:120130] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n15:120130] [ 8] [a32n14:152744] *** Process received signal *** +[a32n14:152744] Signal: Segmentation fault (11) +[a32n14:152744] Signal code: (3) +[a32n14:152744] Failing at address: 0xb170000000000a0 +[a32n14:152744] [ 0] [0x1000000504d8] +[a32n14:152744] [ 1] [0x3fffe991a3d0] +[a32n14:152744] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n14:152744] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n11:156151] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n11:156151] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n07:165088] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n06:179454] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n10:140413] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n10:140413] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n10:140413] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n10:140413] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n10:140413] [14] [b13n18:128542] *** Process received signal *** +[b20n02:122176] *** Process received signal *** +[b20n02:122176] Signal: Segmentation fault (11) +[b20n02:122176] Signal code: (3) +[b20n02:122176] Failing at address: 0xb170000000000a0 +[b20n02:122176] [ 0] [0x1000000504d8] +[b20n02:122176] [ 1] [0x3fffdc23a310] +[b20n02:122176] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n02:122176] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n15:139750] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n15:139750] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n15:139750] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n15:139750] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n10:10926] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n10:10926] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n10:10926] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n10:155399] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n10:155399] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n10:155399] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n10:155399] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n10:155399] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n10:155399] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n10:155399] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n10:155399] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n10:155399] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n02:74430] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n10:169680] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n10:169680] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n10:169680] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n10:169680] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n10:169680] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n10:169680] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n10:169680] *** End of error message *** +[b18n16:124958] *** Process received signal *** +[b18n16:124958] Signal: Segmentation fault (11) +[b18n16:124958] Signal code: (3) +[b18n16:124958] Failing at address: 0xb170000000000a0 +[b18n16:124958] [ 0] [0x1000000504d8] +[b18n16:124958] [ 1] [ 0] [0x1000000504d8] +[a02n10:135853] [ 1] [0x3ffff268c5b0] +[a02n10:135853] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n10:135853] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n10:135853] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n10:135853] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n10:135853] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n10:135853] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n10:135853] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n10:135853] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n10:135853] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n10:135853] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n10:135853] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n10:135853] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n10:135853] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n10:135853] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n18:10724] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n18:10724] [ 8] [b14n08:127443] [ 0] [0x1000000504d8] +[b14n08:127443] [ 1] [0x3fffed508a90] +[b14n08:127443] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n08:127443] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n08:127443] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n01:121745] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n07:172879] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n07:172879] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n07:172879] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n07:172879] [13] [b19n10:120975] *** Process received signal *** +[b19n10:120975] Signal: Segmentation fault (11) +[b19n10:120975] Signal code: (3) +[b19n10:120975] Failing at address: 0xb170000000000a0 +[b19n10:120975] [ 0] [0x1000000504d8] +[b19n10:120975] [ 1] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n12:160190] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n04:130744] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n04:130744] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n04:130744] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n04:130744] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n04:130744] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n02:125373] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n02:125373] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n02:125373] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n02:125373] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n02:125373] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n12:122874] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n12:122874] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n12:122874] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n05:176909] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n05:176909] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n05:176909] [ 8] [b09n09:170654] *** Process received signal *** +[b09n09:170654] Signal: Segmentation fault (11) +[b09n09:170654] Signal code: (3) +[b09n09:170654] Failing at address: 0x14170000000000a0 +[b09n09:170654] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n07:156490] [ 5] [0x3ffffc46ce50] +[a02n08:146118] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n08:146118] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n08:146118] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n08:146118] [ 5] [b15n10:122759] *** Process received signal *** +[b15n10:122759] Signal: Segmentation fault (11) +[b15n10:122759] Signal code: (3) +[b15n10:122759] Failing at address: 0xb170000000000a0 +[b15n10:122759] [ 0] [b17n03:135035] [ 0] [0x1000000504d8] +[b17n03:135035] [ 1] [0x3fffc45844a0] +[b17n03:135035] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n03:135035] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n03:135035] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n07:02839] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n07:02839] [ 7] [0x1000000504d8] +[b17n14:126121] [ 1] [0x3fffca22ac60] +[b17n14:126121] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n14:126121] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n14:126121] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n14:126121] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n14:122441] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n14:122441] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n14:122441] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n14:122441] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n03:147533] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n11:134746] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n11:134746] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n11:134746] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n11:134746] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n11:134746] [10] [0x1000000504d8] +[b18n11:128956] [ 1] [0x3fffc39e5610] +[b18n11:128956] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n11:128956] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n04:136993] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n04:136993] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n04:136993] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n04:136993] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n08:118550] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n08:118550] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n05:173820] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n05:173820] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n05:173820] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n05:173820] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n05:173820] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n05:173820] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n01:123570] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n12:145266] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n12:145266] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n03:119830] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n08:165285] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n08:165285] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n08:165285] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n08:165285] [ 9] [b18n07:125448] *** Process received signal *** +[b18n07:125448] Signal: Segmentation fault (11) +[b18n07:125448] Signal code: (3) +[b18n07:125448] Failing at address: 0x14170000000000a0 +[b18n07:125448] [ 0] [0x1000000504d8] +[b18n07:125448] [ 1] [0x3fffd8904920] +[b18n07:125448] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n07:125448] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n08:124114] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n03:124681] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n03:124681] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n04:177594] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n03:68829] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n03:68829] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n05:05606] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n05:05606] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n05:05606] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n05:05606] *** End of error message *** +[b07n07:163250] *** Process received signal *** +[b07n07:163250] Signal: Segmentation fault (11) +[b07n07:163250] Signal code: (3) +[b07n07:163250] Failing at address: 0xb170000000000a0 +[b07n07:163250] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n09:143896] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n09:143896] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n09:143896] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n09:143896] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n18:121306] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n14:139842] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n14:139842] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n04:117352] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n04:117352] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n04:117352] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n04:117352] [14] [b13n08:144852] *** Process received signal *** +[b13n08:144852] Signal: Segmentation fault (11) +[b13n08:144852] Signal code: (3) +[b13n08:144852] Failing at address: 0xb170000000000a0 +[b13n08:144852] [ 0] [0x1000000504d8] +[b13n08:144852] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n07:156142] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n07:156142] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n07:156142] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n07:156142] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n18:122551] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n18:122551] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n18:122551] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n18:122551] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n15:148163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n15:148163] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n15:148163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n13:138088] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n13:138088] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n13:138088] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n13:138088] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n13:138088] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n16:130376] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n16:130376] *** End of error message *** +[a32n12:164380] *** Process received signal *** +[a32n12:164380] Signal: Segmentation fault (11) +[a32n12:164380] Signal code: (3) +[a32n12:164380] Failing at address: 0xb170000000000a0 +[a32n12:164380] [ 0] [0x1000000504d8] +[a32n12:164380] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n15:32400] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n15:32400] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n15:32400] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n15:32400] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n15:32400] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n15:32400] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n16:147284] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n16:147284] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n16:147284] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n05:121821] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n05:121821] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n05:121821] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n05:121821] *** End of error message *** +[0x1000000504d8] +[b09n01:163021] [ 1] [0x3fffe26a9700] +[b09n01:163021] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n01:163021] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n14:136136] [14] [b02n04:168487] *** Process received signal *** +[b02n04:168487] Signal: Segmentation fault (11) +[b02n04:168487] Signal code: (3) +[b02n04:168487] Failing at address: 0xb170000000000a0 +[b02n04:168487] [ 0] [0x1000000504d8] +[b02n04:168487] [ 1] [0x3fffd77c0570] +[b02n04:168487] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n04:168487] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n11:163261] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n11:163261] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n15:151249] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n17:125805] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n17:125805] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n17:125805] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n17:125805] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n12:178717] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n13:117507] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n13:117507] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n13:117507] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n13:117507] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n15:120060] [ 4] [a28n17:168163] [ 0] [0x1000000504d8] +[a28n17:168163] [ 1] [0x3fffc06841e0] +[a28n17:168163] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n17:168163] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n10:134734] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n10:134734] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n14:150131] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n14:150131] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n14:150131] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n14:150131] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n14:119487] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n14:119487] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n14:119487] [11] [b06n13:166923] *** Process received signal *** +[b06n13:166923] Signal: Segmentation fault (11) +[b06n13:166923] Signal code: (3) +[b06n13:166923] Failing at address: 0x14170000000000a0 +[b06n13:166923] [ 0] [0x1000000504d8] +[b06n13:166923] [ 1] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n13:168159] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n13:168159] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n04:126347] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n04:126347] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n04:126347] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n03:158658] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n03:158658] [ 5] [b22n02:119530] *** Process received signal *** +[b22n02:119530] Signal: Segmentation fault (11) +[b22n02:119530] Signal code: (3) +[b22n02:119530] Failing at address: 0xb170000000000a0 +[b22n02:119530] [ 0] [0x1000000504d8] +[b22n02:119530] [ 1] [0x3fffd7574cc0] +[b22n02:119530] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b22n02:119530] [ 3] [0x1000000504d8] +[a31n11:153419] [ 1] [0x3fffc31b9af0] +[a31n11:153419] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n11:153419] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n11:153419] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n09:161229] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n09:161229] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n09:161229] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n09:161229] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n09:161229] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n11:162473] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n11:162473] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n11:162473] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n11:162473] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n11:162473] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n11:162473] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n11:162473] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n11:162473] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n11:162473] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n11:162473] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n11:162473] *** End of error message *** +[b19n15:124799] *** Process received signal *** +[b19n15:124799] Signal: Segmentation fault (11) +[b19n15:124799] Signal code: (3) +[b19n15:124799] Failing at address: 0x14170000000000a0 +[b19n15:124799] [ 0] [b09n16:173546] *** Process received signal *** +[b09n16:173546] Signal: Segmentation fault (11) +[b09n16:173546] Signal code: (3) +[b09n16:173546] Failing at address: 0xb170000000000a0 +[b09n16:173546] [ 0] [0x1000000504d8] +[b09n16:173546] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n18:164800] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n18:164800] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n18:164800] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n18:164800] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n18:164800] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n16:128442] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n13:123883] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n13:123883] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n13:123883] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n13:123883] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n13:123883] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n18:08430] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n18:08430] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n18:08430] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n18:08430] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n18:08430] [ 9] [b02n05:164941] *** Process received signal *** +[b02n05:164941] Signal: Segmentation fault (11) +[b02n05:164941] Signal code: (3) +[b02n05:164941] Failing at address: 0xb170000000000a0 +[b02n05:164941] [ 0] [0x1000000504d8] +[b02n05:164941] [ 1] [0x3ffff822dcf0] +[b02n05:164941] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n05:164941] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a27n08:54624] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n08:54624] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n12:120128] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n12:120128] *** End of error message *** +[b14n18:131582] *** Process received signal *** +[b14n18:131582] Signal: Segmentation fault (11) +[b14n18:131582] Signal code: (3) +[b14n18:131582] Failing at address: 0xb170000000000a0 +[b13n07:133375] *** Process received signal *** +[b13n07:133375] Signal: Segmentation fault (11) +[b13n07:133375] Signal code: (3) +[b13n07:133375] Failing at address: 0xb170000000000a0 +[b13n07:133375] [ 0] [0x1000000504d8] +[b13n07:133375] [ 1] [0x3fffd32842e0] +[b13n07:133375] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n07:133375] [ 3] [a33n02:167595] *** Process received signal *** +[a33n02:167595] Signal: Segmentation fault (11) +[a33n02:167595] Signal code: (3) +[a33n02:167595] Failing at address: 0x5170000000000a0 +[a33n02:167595] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n17:129930] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n17:129930] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n04:166204] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n04:166204] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n04:166204] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n04:166204] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n04:166204] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n14:133064] *** End of error message *** +[b18n08:131377] *** Process received signal *** +[b18n08:131377] Signal: Segmentation fault (11) +[b18n08:131377] Signal code: (3) +[b18n08:131377] Failing at address: 0x14170000000000a0 +[b18n08:131377] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n17:124163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n17:124163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n17:124163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n17:124163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n17:124163] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n08:116175] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n08:116175] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n08:116175] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n12:132103] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n12:132103] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n12:132103] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n12:132103] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n07:117358] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n07:117358] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n07:117358] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n06:167371] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n06:167371] [ 5] [b14n06:127273] [ 0] [0x1000000504d8] +[b14n06:127273] [ 1] [0x3fffd62ab100] +[b14n06:127273] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n06:127273] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n06:127273] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n06:127273] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n06:127273] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n06:127273] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n16:165925] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n10:162608] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n10:162608] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n10:162608] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n10:162608] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n10:162608] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n10:162608] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n10:162608] [13] [b19n03:123617] [ 0] [0x1000000504d8] +[b19n03:123617] [ 1] [0x3ffff742cb60] +[b19n03:123617] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n03:123617] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n02:176048] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n02:176048] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n02:176048] [12] [0x1000000504d8] +[a02n03:38500] [ 1] [0x3ffff5bc1710] +[a02n03:38500] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n03:38500] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n03:38500] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n03:38500] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n03:38500] [ 6] [0x3fffe699d1d0] +[a29n16:157886] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a29n16:157886] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n16:157886] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n11:160350] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n11:160350] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n11:160350] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n11:160350] [ 9] [b03n02:06560] *** Process received signal *** +[b03n02:06560] Signal: Segmentation fault (11) +[b03n02:06560] Signal code: (3) +[b03n02:06560] Failing at address: 0xb170000000000a0 +[b03n02:06560] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n13:114394] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n13:114394] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n13:114394] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n13:114394] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n16:116522] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n16:116522] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n16:116522] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n16:116522] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n10:142933] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n10:142933] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n10:142933] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n10:142933] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n10:142933] [13] [a02n04:41668] [ 0] [0x1000000504d8] +[a02n04:41668] [ 1] [0x3fffd90a3440] +[a02n04:41668] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n04:41668] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n04:41668] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n04:41668] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n04:41668] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n01:128830] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n01:128830] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n16:127349] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n16:127349] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n16:127349] [ 5] [b07n14:176067] *** Process received signal *** +[b07n14:176067] Signal: Segmentation fault (11) +[b07n14:176067] Signal code: (3) +[b07n14:176067] Failing at address: 0x14170000000000a0 +[b07n14:176067] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n18:179696] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n18:179696] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n18:179696] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n07:180191] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n07:180191] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n07:180191] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n09:123415] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n09:123415] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n12:167675] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n12:167675] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n12:167675] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n12:167675] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n12:167675] [ 9] [0x1000000504d8] +[b08n12:12762] [ 1] [0x3fffd1c2a8f0] +[b08n12:12762] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n12:12762] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n12:12762] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n13:122030] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n13:122030] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n13:122030] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n07:118164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n01:48803] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n01:48803] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n01:48803] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n01:48803] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n01:48803] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n12:122966] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n12:122966] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n01:167301] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n06:01839] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n06:01839] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n17:03843] [10] [a28n10:176307] *** Process received signal *** +[a28n10:176307] Signal: Segmentation fault (11) +[a28n10:176307] Signal code: (3) +[a28n10:176307] Failing at address: 0xb170000000000a0 +[a28n10:176307] [ 0] [0x1000000504d8] +[a28n10:176307] [ 1] [0x3fffd8a514f0] +[a28n10:176307] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n10:176307] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n10:176307] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n10:176307] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n10:176307] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n10:176307] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n10:176307] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n14:133635] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n14:133635] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n06:126379] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n17:15871] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n17:15871] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n17:15871] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n17:15871] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n17:15871] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n17:15871] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n08:147819] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n08:147819] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n08:147819] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n14:169520] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n14:169520] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n14:169520] [13] [0x3fffdfc7e750] +[b05n08:08953] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n08:08953] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n08:08953] [ 4] [0x1000000504d8] +[b17n18:126907] [ 1] [0x3fffc42922f0] +[b17n18:126907] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n18:126907] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n18:126907] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n18:126907] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n06:143010] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n06:143010] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n06:143010] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n06:143010] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n06:143010] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n01:141814] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n01:141814] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n01:141814] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n14:128868] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n14:128868] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n14:128868] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n14:128868] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n14:128868] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n17:149906] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n17:149906] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n17:149906] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n13:176364] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n13:176364] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n13:176364] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n13:176364] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n13:176364] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n13:176364] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n15:177055] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n15:177055] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n15:177055] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n07:132086] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n07:132086] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n07:132086] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n07:132086] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n07:132086] [14] [0x3fffdbaf1d00] +[a33n04:169909] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n04:169909] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n04:169909] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n07:138204] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n07:138204] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n07:138204] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n07:138204] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n07:138204] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n05:135163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n05:135163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n05:135163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n17:150930] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n02:164491] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n02:164491] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n11:137870] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n11:137870] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n07:131670] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n07:131670] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n07:131670] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n07:131670] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n07:131670] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n07:131670] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n16:01570] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n16:01570] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n16:01570] [ 9] [b18n01:128155] *** Process received signal *** +[b18n01:128155] Signal: Segmentation fault (11) +[b18n01:128155] Signal code: (3) +[b18n01:128155] Failing at address: 0x14170000000000a0 +[b18n01:128155] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n03:10289] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n03:10289] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n03:10289] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n03:10289] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n02:166744] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n02:166744] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n02:166744] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n02:166744] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n02:166744] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n02:166744] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n02:166744] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n09:121751] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n09:121751] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n13:123705] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n13:123705] [ 6] [a28n11:01174] *** Process received signal *** +[a28n11:01174] Signal: Segmentation fault (11) +[a28n11:01174] Signal code: (3) +[a28n11:01174] Failing at address: 0x14170000000000a0 +[b11n11:152985] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n11:152985] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n18:28219] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n18:28219] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n18:28219] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n18:28219] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n18:28219] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n18:28219] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n18:28219] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n01:130408] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n01:130408] *** End of error message *** +[0x1000000504d8] +[b11n14:161620] [ 1] [0x3fffdfb68a40] +[b11n14:161620] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n11:171961] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n11:171961] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n11:171961] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n08:124723] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n08:124723] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n08:124723] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n08:124723] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n08:124723] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n09:173753] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n09:173753] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n09:173753] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n09:173753] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n12:148378] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n12:148378] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n12:148378] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n16:122491] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n16:122491] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n01:173781] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n10:130579] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n02:143965] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n02:143965] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n02:143965] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n16:153292] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n16:153292] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n15:85943] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n15:85943] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n12:11142] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n12:11142] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n09:119546] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n09:119546] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n09:119546] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n09:119546] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n12:156914] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n12:156914] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n12:156914] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n12:156914] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n12:156914] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n02:120189] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n02:120189] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n02:120189] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n02:120189] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n02:120189] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n17:119579] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n17:119579] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n17:119579] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n17:119579] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n04:175996] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n04:175996] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n04:175996] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n04:175996] [ 9] [0x3fffecdb7f60] +[b12n01:145730] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b12n01:145730] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b12n01:145730] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b12n01:145730] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n15:148583] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n15:148583] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n15:148583] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n15:148583] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n02:163318] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n02:163318] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n02:163318] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n02:163318] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n09:123810] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n07:125471] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n07:125471] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n08:163163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n08:163163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n08:163163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n08:163163] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n08:163163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n08:163163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n08:163163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n08:163163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n08:163163] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n10:122206] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n10:122206] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n10:122206] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n10:122206] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n17:164563] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n17:164563] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n17:164563] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n11:147713] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n11:147713] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n07:127357] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n07:127357] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n12:115639] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n12:115639] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n12:115639] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n08:156368] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n08:156368] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n08:156368] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n08:156368] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n08:156368] [13] [a32n13:163195] [ 7] [0x1000000504d8] +[b07n13:165258] [ 1] [0x3fffc88e33d0] +[b07n13:165258] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n13:165258] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n13:165258] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n05:169785] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n08:121686] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n08:121686] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n08:121686] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n15:137005] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n15:137005] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n15:137005] [ 9] [b06n12:177117] *** Process received signal *** +[b06n12:177117] Signal: Segmentation fault (11) +[b06n12:177117] Signal code: (3) +[b06n12:177117] Failing at address: 0xb170000000000a0 +[b06n12:177117] [ 0] [0x1000000504d8] +[b06n12:177117] [ 1] [0x3fffe003b250] +[b06n12:177117] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n12:177117] [ 3] [a30n08:163096] [ 0] [0x1000000504d8] +[a30n08:163096] [ 1] [0x3fffc50d7770] +[a30n08:163096] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n08:163096] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n08:163096] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n08:163096] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n08:163096] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n11:120239] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n11:120239] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n11:120239] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n02:149230] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n11:175593] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n11:175593] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n11:02683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n11:02683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n11:02683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n11:02683] [10] [a35n17:146797] *** Process received signal *** +[a35n17:146797] Signal: Segmentation fault (11) +[a35n17:146797] Signal code: (3) +[a35n17:146797] Failing at address: 0x5170000000000a0 +[a35n17:146797] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n17:126524] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n17:126524] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n17:126524] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n17:126524] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n03:168933] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n15:123038] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n15:123038] [ 6] [0x1000000504d8] +[a35n01:138007] [ 1] [0x3fffc9fbab90] +[a35n01:138007] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n01:138007] [ 3] [b04n06:10222] [ 0] [0x1000000504d8] +[b04n06:10222] [ 1] [0x3fffd5593d70] +[b04n06:10222] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n06:10222] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n06:10222] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n06:177007] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n06:177007] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n14:163549] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n14:163549] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n03:135161] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n03:135161] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n03:135161] *** End of error message *** +[a28n02:178476] *** Process received signal *** +[a28n02:178476] Signal: Segmentation fault (11) +[a28n02:178476] Signal code: (3) +[a28n02:178476] Failing at address: 0x14170000000000a0 +[a28n02:178476] [ 0] [0x1000000504d8] +[a28n02:178476] [ 1] [0x3fffe4557e70] +[a28n02:178476] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n02:178476] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n03:136664] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n03:136664] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n15:161753] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n12:154344] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n12:154344] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n01:65724] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n01:65724] [ 5] [a34n01:164960] [ 0] [0x1000000504d8] +[a34n01:164960] [ 1] [0x3fffc8b637a0] +[a34n01:164960] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n01:164960] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n01:164960] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n06:154620] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n06:154620] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n05:162568] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n05:162568] [ 5] [b01n04:169702] *** Process received signal *** +[b01n04:169702] Signal: Segmentation fault (11) +[b01n04:169702] Signal code: (3) +[b01n04:169702] Failing at address: 0xb170000000000a0 +[b01n04:169702] [ 0] [0x1000000504d8] +[b01n04:169702] [ 1] [0x3fffd62b4860] +[b01n04:169702] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n04:169702] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n03:01353] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n03:01353] *** End of error message *** +[0x3fffea1b9600] +[a35n11:148671] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n11:148671] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n11:148671] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n11:148671] [ 5] [b15n02:122558] *** Process received signal *** +[b15n02:122558] Signal: Segmentation fault (11) +[b15n02:122558] Signal code: (3) +[b15n02:122558] Failing at address: 0x14170000000000a0 +[b15n02:122558] [ 0] [0x1000000504d8] +[b15n02:122558] [ 1] [0x3fffdd953550] +[b15n02:122558] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n02:122558] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n09:147793] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n09:147793] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n09:147793] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n09:147793] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n09:147793] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n09:147793] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n17:119465] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n17:119465] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n06:133955] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n06:133955] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n06:133955] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n02:123702] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n02:123702] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n02:123702] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n05:135120] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n05:135120] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n05:135120] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n05:135120] [10] [a32n03:151882] [ 0] [0x1000000504d8] +[a32n03:151882] [ 1] [0x3fffd28bfec0] +[a32n03:151882] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n03:151882] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n03:151882] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n03:125242] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n03:125242] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n03:125242] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n03:125242] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n06:130540] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n06:130540] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n17:163816] [ 7] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n05:131263] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n05:131263] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n05:131263] [14] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n17:118572] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n17:118572] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n17:118572] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n01:179259] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n01:141343] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n01:141343] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n01:141343] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n01:141343] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n03:153852] [ 5] [0x3fffc209b1a0] +[a36n02:147882] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n02:147882] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n02:147882] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n02:147882] [ 5] [b21n18:118683] [ 0] [0x1000000504d8] +[b21n18:118683] [ 1] [0x3fffcea563f0] +[b21n18:118683] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n18:118683] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n18:118683] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n18:118683] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n18:118683] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n16:121930] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n16:121930] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n16:121930] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n16:121930] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n16:121930] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n08:172429] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n08:172429] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n08:172429] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n08:172429] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n08:172429] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n08:172429] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n08:172429] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n08:172429] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n08:172429] *** End of error message *** +[b14n03:130348] *** Process received signal *** +[b14n03:130348] Signal: Segmentation fault (11) +[b14n03:130348] Signal code: (3) +[b14n03:130348] Failing at address: 0xb170000000000a0 +[b14n03:130348] [ 0] [0x1000000504d8] +[b14n03:130348] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n08:164749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n08:164749] [10] [b03n04:04846] *** Process received signal *** +[b03n04:04846] Signal: Segmentation fault (11) +[b03n04:04846] Signal code: (3) +[b03n04:04846] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n06:162916] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n06:162916] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n06:162916] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n06:162916] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n06:171500] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n06:171500] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n06:171500] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n02:139026] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n02:139026] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n02:139026] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n02:139026] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n02:139026] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n06:163674] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n06:163674] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n06:163674] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n06:163674] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n06:120732] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n06:120732] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n06:120732] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n06:120732] [14] [b14n01:128001] *** Process received signal *** +[b14n01:128001] Signal: Segmentation fault (11) +[b14n01:128001] Signal code: (3) +[b14n01:128001] Failing at address: 0x14170000000000a0 +[b14n01:128001] [ 0] [0x1000000504d8] +[b14n01:128001] [ 1] [0x3fffe4f27ac0] +[b14n01:128001] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n01:128001] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n12:138734] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n12:138734] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n12:138734] [10] [ 0] [0x1000000504d8] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n17:123237] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n17:123237] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n17:123237] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n17:123237] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n17:123237] [10] [b10n16:164553] *** Process received signal *** +[b10n16:164553] Signal: Segmentation fault (11) +[b10n16:164553] Signal code: (3) +[b10n16:164553] Failing at address: 0xb170000000000a0 +[b10n16:164553] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n11:160795] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n11:160795] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n11:160795] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n11:160795] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n01:165123] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n01:165123] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n01:165123] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n04:126858] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n04:126858] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n05:126778] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n05:126778] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n05:126778] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n05:126778] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n07:124348] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n07:124348] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n07:124348] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n07:124348] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n07:124348] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n09:155759] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n09:155759] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n09:155759] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n09:155759] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n09:155759] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n09:155759] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n06:157788] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n06:157788] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n06:157788] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n06:157788] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n06:157788] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n06:157788] *** End of error message *** +[b07n10:16970] *** Process received signal *** +[b07n10:16970] Signal: Segmentation fault (11) +[b07n10:16970] Signal code: (3) +[b07n10:16970] Failing at address: 0xb170000000000a0 +[b07n10:16970] [ 0] [0x1000000504d8] +[b07n10:16970] [ 1] [0x3fffc3075d70] +[b07n10:16970] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n10:16970] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n11:117473] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n11:117473] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n11:117473] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n11:117473] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n11:117473] *** End of error message *** +[b01n12:178602] Signal: Segmentation fault (11) +[b01n12:178602] Signal code: (3) +[b01n12:178602] Failing at address: 0x14170000000000a0 +[b01n12:178602] [b01n11:03982] [ 0] [0x1000000504d8] +[b01n11:03982] [ 1] [0x3ffff5938b00] +[b01n11:03982] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n11:03982] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n11:120583] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n11:120583] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n11:120583] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n11:120583] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n11:120583] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n07:160801] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n07:160801] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n07:160801] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n07:160801] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n06:117528] *** End of error message *** +[a28n18:179167] [ 0] [0x1000000504d8] +[a28n18:179167] [ 1] [0x3fffc1371bc0] +[a28n18:179167] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n18:179167] [ 3] [b17n13:123514] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n04:158631] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n04:158631] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n04:158631] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n04:158631] *** End of error message *** +[a30n17:176411] *** Process received signal *** +[a30n17:176411] Signal: Segmentation fault (11) +[a30n17:176411] Signal code: (3) +[a30n17:176411] Failing at address: 0xb170000000000a0 +[a30n17:176411] [ 0] [b13n11:134520] *** Process received signal *** +[b13n11:134520] Signal: Segmentation fault (11) +[b13n11:134520] Signal code: (3) +[b13n11:134520] Failing at address: 0x14170000000000a0 +[b08n14:178322] *** Process received signal *** +[b08n14:178322] Signal: Segmentation fault (11) +[b08n14:178322] Signal code: (3) +[b08n14:178322] Failing at address: 0xb170000000000a0 +[b08n14:178322] [ 0] [a33n17:168153] *** Process received signal *** +[a33n17:168153] Signal: Segmentation fault (11) +[a33n17:168153] Signal code: (3) +[a33n17:168153] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n04:44759] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n04:44759] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n04:44759] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n03:154292] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n03:154292] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n09:119631] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n09:119631] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n09:119631] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n09:119631] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n13:171106] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n13:171106] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n13:171106] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n05:139437] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n05:139437] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n05:139437] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n07:168960] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n07:168960] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n10:11492] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n10:11492] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n10:11492] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n10:11492] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n01:163390] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n05:174778] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n05:174778] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n05:174778] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n05:174778] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n05:174778] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n10:118389] [ 5] [0x1000000504d8] +[b03n03:06822] [ 1] [0x3ffff481e980] +[b03n03:06822] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n03:06822] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n03:06822] [ 4] [b04n01:96752] *** Process received signal *** +[b04n01:96752] Signal: Segmentation fault (11) +[b04n01:96752] Signal code: (3) +[b04n01:96752] Failing at address: 0x5170000000000a0 +[b04n01:96752] [ 0] [0x1000000504d8] +[b04n01:96752] [ 1] [0x3fffe056a250] +[b04n01:96752] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n01:96752] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n12:123562] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n12:123562] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n12:123562] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n12:123562] [ 9] [a30n10:169206] *** Process received signal *** +[a30n10:169206] Signal: Segmentation fault (11) +[a30n10:169206] Signal code: (3) +[a30n10:169206] Failing at address: 0x5170000000000a0 +[a30n10:169206] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n13:123051] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n13:123051] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n13:123051] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n13:123051] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n13:123051] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n05:139912] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n05:139912] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n05:139912] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n05:139912] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n12:160963] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n12:160963] *** End of error message *** +[0x1000000504d8] +[a30n13:154234] [ 1] [0x3fffcb7e6e40] +[a30n13:154234] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n13:154234] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n13:154234] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n13:154234] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n13:154234] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n13:154234] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n13:154234] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n13:154234] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n13:154234] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n05:120227] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n05:120227] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n05:120227] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n05:120227] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n05:120227] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n03:172649] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n03:172649] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n13:133626] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n13:133626] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n13:133626] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n13:133626] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n01:170766] *** End of error message *** +[0x1000000504d8] +[b11n16:154182] [ 1] [0x3ffff2d13220] +[b11n16:154182] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n16:154182] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n16:154182] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n08:176683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n08:176683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n08:176683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n08:176683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n08:176683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n08:176683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n08:176683] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n15:120129] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n15:120129] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n15:120129] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n15:120129] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n14:152747] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n14:152747] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n07:165087] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n07:165087] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n06:179454] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n10:140414] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n10:140414] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n10:140414] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n10:140414] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n10:140414] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n10:140414] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n10:140414] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n11:156151] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n11:156151] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n11:156151] [ 8] [b13n18:128542] Signal: Segmentation fault (11) +[b13n18:128542] Signal code: (3) +[b13n18:128542] Failing at address: 0xb170000000000a0 +[b13n18:128542] [ 0] [0x1000000504d8] +[b13n18:128542] [ 1] [0x3fffc50bfd90] +[b13n18:128542] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n18:128542] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n18:128542] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n02:122177] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n02:122177] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n10:155399] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n10:155399] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n10:155399] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n10:10925] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n10:10925] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n15:139749] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n15:139749] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n15:139749] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n15:139749] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n15:139749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n15:139749] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n15:139749] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n15:139749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n15:139749] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n02:74430] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n02:74430] [13] [0x1000000504d8] +[b18n16:124957] [ 1] [0x3ffffddb7d10] +[b18n16:124957] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n16:124957] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n16:124957] [ 4] [ 0] [0x1000000504d8] +[a02n10:135854] [ 1] [0x3fffdc19fce0] +[a02n10:135854] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n10:135854] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n10:135854] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n10:135854] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n10:135854] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n10:135854] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n10:135854] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n10:135854] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n10:135854] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n10:135854] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n10:135854] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n10:135854] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n10:135854] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n10:135854] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n08:127442] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n08:127442] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n08:127442] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n18:10724] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n07:172880] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n07:172880] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n07:172880] [ 9] [b19n01:121746] *** End of error message *** +[b19n10:120974] *** Process received signal *** +[b19n10:120974] Signal: Segmentation fault (11) +[b19n10:120974] Signal code: (3) +[b19n10:120974] Failing at address: 0xb170000000000a0 +[b19n10:120974] [ 0] [0x1000000504d8] +[b19n10:120974] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n02:125374] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n02:125374] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n04:130745] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n04:130745] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n04:130745] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n05:176909] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n05:176909] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n12:122873] [ 6] [b09n09:170655] *** Process received signal *** +[b09n09:170655] Signal: Segmentation fault (11) +[b09n09:170655] Signal code: (3) +[b09n09:170655] Failing at address: 0xb170000000000a0 +[b09n09:170655] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n08:146117] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n08:146117] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n08:146117] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n07:156489] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n07:156489] [ 8] [b17n03:135034] *** Process received signal *** +[b17n03:135034] Signal: Segmentation fault (11) +[b17n03:135034] Signal code: (3) +[b17n03:135034] Failing at address: 0x14170000000000a0 +[b17n03:135034] [ 0] [0x1000000504d8] +[b17n03:135034] [ 1] [0x3ffff1d6d440] +[b17n03:135034] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n03:135034] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n03:135034] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n03:124681] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n12:145265] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n12:145265] *** End of error message *** +[0x1000000504d8] +[b17n14:126122] [ 1] [0x3fffd7504800] +[b17n14:126122] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n14:126122] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n14:126122] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n14:126122] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n11:134745] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n11:134745] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n11:134745] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n14:122440] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n14:122440] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n14:122440] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n04:136994] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n04:136994] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n04:136994] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n04:136994] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n04:136994] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n08:118551] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n08:118551] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n05:173819] *** End of error message *** +[b15n01:123572] *** Process received signal *** +[b15n01:123572] Signal: Segmentation fault (11) +[b15n01:123572] Signal code: (3) +[b15n01:123572] Failing at address: 0x14170000000000a0 +[b15n01:123572] [ 0] [b15n03:119827] *** Process received signal *** +[b15n03:119827] Signal: Segmentation fault (11) +[b15n03:119827] Signal code: (3) +[b15n03:119827] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n08:165284] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n08:165284] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n08:124115] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n07:125448] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n07:125448] [ 5] [0x1000000504d8] +[b15n10:122760] [ 1] [0x3fffdea663f0] +[b15n10:122760] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n10:122760] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n10:122760] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n07:02839] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n07:02839] [ 9] [0x1000000504d8] +[b18n11:128957] [ 1] [0x3fffc01df480] +[b18n11:128957] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n11:128957] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n11:128957] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n11:128957] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n11:128957] [ 6] [0x1000000504d8] +[b09n04:177595] [ 1] [0x3ffff53e4ed0] +[b09n04:177595] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n04:177595] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n04:177595] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n04:177595] [ 5] [0x1000000504d8] +[b07n07:163250] [ 1] [0x3ffff0641eb0] +[b07n07:163250] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n07:163250] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n07:163250] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n09:143896] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n18:121307] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n18:121307] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n18:121307] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n18:121307] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n18:121307] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n04:117351] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n04:117351] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n04:117351] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n04:117351] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n04:117351] [13] [b13n08:144851] [ 0] [0x1000000504d8] +[b13n08:144851] [ 1] [0x3fffe1d19660] +[b13n08:144851] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n08:144851] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n08:144851] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n07:156143] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n07:156143] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n07:156143] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n07:156143] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n18:122552] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n18:122552] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n18:122552] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n15:148164] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n15:148164] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n13:138087] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n13:138087] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n13:138087] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n13:138087] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n15:32398] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n15:32398] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n15:32398] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n15:32398] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n15:32398] [14] [0x1000000504d8] +[a32n12:164379] [ 1] [0x3fffe2b83f10] +[a32n12:164379] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n12:164379] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n12:164379] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n12:164379] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n05:121822] *** End of error message *** +[b19n16:130373] *** Process received signal *** +[b19n16:130373] Signal: Segmentation fault (11) +[b19n16:130373] Signal code: (3) +[b19n16:130373] Failing at address: 0xb170000000000a0 +[b19n16:130373] [ 0] [a32n16:147283] *** Process received signal *** +[a32n16:147283] Signal: Segmentation fault (11) +[a32n16:147283] Signal code: (3) +[a32n16:147283] Failing at address: 0xb170000000000a0 +[a32n16:147283] [ 0] [0x1000000504d8] +[a32n16:147283] [ 1] [0x3fffee8e51d0] +[a32n16:147283] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n16:147283] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n16:147283] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n16:147283] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n16:147283] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n16:147283] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n16:147283] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n16:147283] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n16:147283] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n16:147283] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n16:147283] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n16:147283] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n16:147283] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n16:147283] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n14:136136] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n01:163021] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n04:168488] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n04:168488] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n04:168488] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n04:168488] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n04:168488] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n04:168488] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n04:168488] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n04:168488] [13] [a30n11:163258] *** Process received signal *** +[a30n11:163258] Signal: Segmentation fault (11) +[a30n11:163258] Signal code: (3) +[a30n11:163258] Failing at address: 0xb170000000000a0 +[a30n11:163258] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n15:151248] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n15:151248] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n15:151248] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n15:151248] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n15:151248] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n12:178716] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n12:178716] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n12:178716] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n12:178716] [ 8] [b17n17:125807] *** Process received signal *** +[b17n17:125807] Signal: Segmentation fault (11) +[b17n17:125807] Signal code: (3) +[b17n17:125807] Failing at address: 0x14170000000000a0 +[b17n17:125807] [ 0] [0x1000000504d8] +[b17n17:125807] [ 1] [0x3fffda4c8ad0] +[b17n17:125807] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n17:125807] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n13:117508] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n17:168162] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n17:168162] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n17:168162] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n17:168162] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n17:168162] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n17:168162] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n15:120060] [ 5] [b13n16:128441] *** Process received signal *** +[b13n16:128441] Signal: Segmentation fault (11) +[b13n16:128441] Signal code: (3) +[b13n16:128441] Failing at address: 0x14170000000000a0 +[b13n16:128441] [ 0] [0x1000000504d8] +[b13n16:128441] [ 1] [0x3ffff9e42610] +[b13n16:128441] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n16:128441] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n14:150130] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n14:150130] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n14:150130] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n14:150130] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n14:119486] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n14:119486] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n14:119486] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n14:119486] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n14:119486] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n13:168158] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n13:168158] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n13:168158] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n13:168158] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n04:126348] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n04:126348] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n04:126348] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n13:166922] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n03:158659] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n03:158659] [ 6] [a31n11:153418] *** Process received signal *** +[a31n11:153418] Signal: Segmentation fault (11) +[a31n11:153418] Signal code: (3) +[a31n11:153418] Failing at address: 0x14170000000000a0 +[a31n11:153418] [ 0] [0x1000000504d8] +[a31n11:153418] [ 1] [0x3fffd35c5100] +[a31n11:153418] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n11:153418] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n02:119531] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n02:119531] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n09:161228] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n09:161228] *** End of error message *** +[b19n15:124798] *** Process received signal *** +[b19n15:124798] Signal: Segmentation fault (11) +[b19n15:124798] Signal code: (3) +[b19n15:124798] Failing at address: 0xb170000000000a0 +[b19n15:124798] [ 0] [0x1000000504d8] +[b19n15:124798] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n16:173547] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n16:173547] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n16:173547] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n13:123884] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n13:123884] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n13:123884] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n12:132103] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n12:132103] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n12:132103] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n18:164799] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n18:164799] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n18:164799] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n18:164799] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n18:164799] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n18:164799] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n05:164942] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n05:164942] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n05:164942] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n05:164942] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n05:164942] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n05:164942] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n05:164942] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n18:08431] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n18:08431] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n18:08431] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n18:08431] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a27n08:54623] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n08:54623] [ 6] [b14n18:131582] [ 0] [0x1000000504d8] +[b14n18:131582] [ 1] [0x3fffec87b560] +[b14n18:131582] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n18:131582] [ 3] [ 0] [0x1000000504d8] +[a33n02:167595] [ 1] [0x3ffff5c4d740] +[a33n02:167595] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n02:167595] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n12:120127] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n12:120127] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n12:120127] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n12:120127] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n17:129929] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n17:129929] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n10:162607] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n10:162607] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n10:162607] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n10:162607] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n10:162607] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n10:162607] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n10:162607] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n10:162607] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n10:162607] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n10:162607] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n04:166205] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n04:166205] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n04:166205] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n04:166205] [13] [0x1000000504d8] +[b18n08:131377] [ 1] [0x3fffed5a0380] +[b18n08:131377] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n08:131377] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n08:131377] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n14:133063] [b13n10:134730] *** Process received signal *** +[b13n10:134730] Signal: Segmentation fault (11) +[b13n10:134730] Signal code: (3) +[b13n10:134730] Failing at address: 0x14170000000000a0 +[b13n10:134730] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n06:167370] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n06:167370] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n06:167370] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n06:167370] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n08:116175] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n08:116175] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n08:116175] [12] [b06n16:165923] *** Process received signal *** +[b06n16:165923] Signal: Segmentation fault (11) +[b06n16:165923] Signal code: (3) +[b06n16:165923] Failing at address: 0xb170000000000a0 +[b06n16:165923] [ 0] [0x1000000504d8] +[b06n16:165923] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n06:127272] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n06:127272] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n06:127272] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n06:127272] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n06:127272] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n06:127272] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n06:127272] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n17:124164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n17:124164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n17:124164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n17:124164] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n03:123617] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n03:123617] [ 5] [b13n07:133376] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n07:133376] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n07:117358] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n07:117358] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n02:176047] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n02:176047] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n02:176047] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n02:176047] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n02:176047] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n02:176047] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n02:176047] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n03:38499] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n03:38499] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n03:38499] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a29n16:157887] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n16:157887] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n16:157887] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n11:160349] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n11:160349] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n11:160349] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n11:160349] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n11:160349] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n13:114395] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n10:142932] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n10:142932] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n16:116523] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n16:116523] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n01:128831] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n01:128831] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n04:41669] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n04:41669] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n04:41669] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n04:41669] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n04:41669] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n04:41669] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n16:127350] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n16:127350] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n16:127350] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n16:127350] [ 9] [0x1000000504d8] +[b07n14:176067] [ 1] [0x3ffffbd35200] +[b07n14:176067] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n14:176067] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n14:176067] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n09:123416] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n09:123416] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n18:179697] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n18:179697] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n18:179697] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n18:179697] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n07:118164] [ 5] [b03n02:06559] *** Process received signal *** +[b03n02:06559] Signal: Segmentation fault (11) +[b03n02:06559] Signal code: (3) +[b03n02:06559] Failing at address: 0x14170000000000a0 +[b03n02:06559] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n01:48802] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n01:48802] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n01:48802] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n01:48802] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n12:12763] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n12:12763] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n12:167674] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n12:167674] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n12:167674] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n12:167674] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n12:167674] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n12:167674] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n12:167674] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n12:167674] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n12:167674] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n12:167674] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n07:180191] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n07:180191] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n13:122030] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n13:122030] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n12:122966] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n17:149906] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n14:128867] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n14:128867] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n14:128867] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n14:128867] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n14:128867] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n17:03843] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b16n14:133636] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b16n14:133636] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b16n14:133636] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b16n14:133636] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b16n14:133636] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n15:177054] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n15:177054] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n15:177054] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n06:126379] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n05:135162] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n05:135162] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n05:135162] [11] [b05n08:08952] *** Process received signal *** +[b05n08:08952] Signal: Segmentation fault (11) +[b05n08:08952] Signal code: (3) +[b05n08:08952] Failing at address: 0x14170000000000a0 +[b05n08:08952] [ 0] [0x1000000504d8] +[b05n08:08952] [ 1] [0x3ffffd6d22c0] +[b05n08:08952] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n08:08952] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n18:126906] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n18:126906] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n14:169521] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n06:143008] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n06:143008] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n06:143008] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n06:143008] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n06:01840] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n01:141814] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n01:141814] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n01:141814] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n01:167300] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n01:167300] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n13:176363] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n13:176363] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n13:176363] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n13:176363] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n13:176363] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n10:176306] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n10:176306] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n10:176306] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n10:176306] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n10:176306] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n07:132087] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n07:132087] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n07:132087] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n08:147819] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n08:147819] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n08:147819] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n08:147819] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n04:169910] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n04:169910] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n07:138205] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n07:138205] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n07:138205] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n17:150931] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n17:150931] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n17:150931] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n17:150931] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n02:164492] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n02:164492] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n02:164492] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n02:164492] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n11:137869] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n11:137869] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n11:137869] [ 9] [0x1000000504d8] +[b18n01:128155] [ 1] [0x3fffdf5f0010] +[b18n01:128155] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n01:128155] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n01:128155] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n03:10288] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n03:10288] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n03:10288] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n03:10288] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n03:10288] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n07:131669] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n07:131669] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n07:131669] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n07:131669] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n07:131669] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n07:131669] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n07:131669] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n16:01568] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n16:01568] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n16:01568] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n16:01568] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n16:01568] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n02:166744] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n02:166744] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n02:166744] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n02:166744] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n09:121752] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n09:121752] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n09:121752] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n09:121752] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n09:121752] [13] [a28n11:01174] [ 0] [0x1000000504d8] +[a28n11:01174] [ 1] [0x3fffd3068a90] +[a28n11:01174] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n11:01174] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n11:01174] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n13:123705] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n13:123705] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n13:123705] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n11:152985] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n11:152985] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n18:28220] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n18:28220] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n14:161620] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n11:171962] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n11:171962] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n11:171962] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n11:171962] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n11:171962] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n11:171962] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n11:171962] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n09:173752] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n09:173752] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n09:173752] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n09:173752] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n09:173752] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n16:122492] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n16:122492] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n16:122492] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n12:148378] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n12:148378] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n02:143965] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n02:143965] [11] [b05n12:11143] [ 0] [0x1000000504d8] +[b05n12:11143] [ 1] [0x3fffe10593d0] +[b05n12:11143] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b05n12:11143] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n12:11143] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n12:11143] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n12:11143] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n12:156915] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n12:156915] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n12:156915] [11] [a30n16:153288] *** Process received signal *** +[a30n16:153288] Signal: Segmentation fault (11) +[a30n16:153288] Signal code: (3) +[a30n16:153288] Failing at address: 0xb170000000000a0 +[b21n02:120188] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n17:119580] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n17:119580] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n04:175997] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n04:175997] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n04:175997] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n04:175997] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n15:148582] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n15:148582] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n15:148582] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n15:148582] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n01:145731] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n01:145731] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n01:145731] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n02:163319] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n09:123810] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n08:163164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n08:163164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n08:163164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n08:163164] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n08:163164] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n17:164562] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n17:164562] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n17:164562] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n17:164562] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n17:164562] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n12:115640] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n12:115640] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n12:115640] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n12:115640] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n07:127357] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n07:127357] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n07:127357] [13] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n08:156369] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n08:156369] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n08:156369] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n08:156369] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n08:121686] *** End of error message *** +[0x3fffd573de60] +[b07n13:165259] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n13:163195] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n13:163195] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n15:137004] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n15:137004] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n15:137004] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n15:137004] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n12:177116] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n12:177116] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n12:177116] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n12:177116] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n12:177116] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n12:177116] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n12:177116] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n12:177116] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n12:177116] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n12:177116] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n12:177116] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n08:163096] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n08:163096] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n08:163096] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n08:163096] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n11:120238] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n11:120238] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n11:120238] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n02:149230] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n02:149230] [ 6] [0x3fffca2ec240] +[b03n11:175594] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n11:175594] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n11:175594] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n11:175594] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n11:175594] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n11:02682] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n11:02682] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n11:02682] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n11:02682] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n11:02682] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n17:126524] *** End of error message *** +[a35n17:146796] [ 0] [0x1000000504d8] +[a35n17:146796] [ 1] [0x3fffe4a54130] +[a35n17:146796] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n17:146796] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n17:146796] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n03:168933] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n03:168933] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n03:168933] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n03:168933] [10] [b07n06:177006] Signal: Segmentation fault (11) +[b07n06:177006] Signal code: (3) +[b07n06:177006] Failing at address: 0x14170000000000a0 +[b07n06:177006] [ 0] [0x1000000504d8] +[b07n06:177006] [ 1] [0x3ffff1f34b00] +[b07n06:177006] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n06:177006] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n06:177006] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n15:123037] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n15:123037] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n01:138006] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n01:138006] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n01:138006] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n14:163549] [14] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n06:10223] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n06:10223] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n03:135162] [ 4] [a28n02:178477] [ 0] [0x1000000504d8] +[a28n02:178477] [ 1] [0x3fffc7bf5960] +[a28n02:178477] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n02:178477] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n15:161753] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n15:161753] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n15:161753] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n03:136665] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n03:136665] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n03:136665] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n03:136665] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a26n01:65725] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a26n01:65725] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n01:65725] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n03:125243] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n03:125243] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n03:125243] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n03:125243] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n03:125243] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n06:154619] [ 4] [ 0] [0x1000000504d8] +[b11n05:162569] [ 1] [0x3fffd4c41b70] +[b11n05:162569] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n05:162569] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n05:162569] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n05:162569] [ 5] [b01n04:169703] [ 0] [0x1000000504d8] +[b01n04:169703] [ 1] [0x3fffe11f42a0] +[b01n04:169703] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n04:169703] [ 3] [b01n03:01355] *** Process received signal *** +[b01n03:01355] Signal: Segmentation fault (11) +[b01n03:01355] Signal code: (3) +[b01n03:01355] Failing at address: 0xb170000000000a0 +[ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n11:148672] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n02:122559] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n09:147794] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n09:147794] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n09:147794] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n09:147794] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n09:147794] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n09:147794] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n17:119464] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n17:119464] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n17:119464] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n17:119464] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n17:119464] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n17:119464] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n06:133956] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n06:133956] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n06:133956] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n05:135119] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n05:135119] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n05:135119] [11] [b19n02:123703] *** Process received signal *** +[b19n02:123703] Signal: Segmentation fault (11) +[b19n02:123703] Signal code: (3) +[b19n02:123703] Failing at address: 0x14170000000000a0 +[b19n02:123703] [ 0] [0x1000000504d8] +[b19n02:123703] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n03:151882] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n06:130541] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n06:130541] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n06:130541] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n06:130541] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n01:164960] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n17:163816] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n05:131263] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n17:118571] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n17:118571] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n17:118571] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n17:118571] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n01:179260] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n01:179260] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n02:147881] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n02:147881] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n02:147881] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n02:147881] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n02:147881] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n01:141344] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n01:141344] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n01:141344] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n01:141344] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n01:141344] [10] [0x3fffc43fcff0] +[b14n03:130348] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n03:130348] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n03:153852] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n03:153852] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n03:153852] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n03:153852] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n03:153852] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n08:164748] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n08:164748] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n08:164748] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n08:164748] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n08:164748] [13] [b03n04:04846] [ 0] [0x1000000504d8] +[b03n04:04846] [ 1] [0x3ffffc689010] +[b03n04:04846] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n04:04846] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n18:118684] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n18:118684] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n18:118684] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n18:118684] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n18:118684] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n18:118684] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n08:172431] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n08:172431] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n08:172431] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n08:172431] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n08:172431] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n06:162915] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n06:162915] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n06:162915] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n06:162915] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n06:163675] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n06:163675] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n06:163675] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n06:163675] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n06:171501] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n06:171501] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n06:171501] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n06:171501] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n06:171501] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n01:128002] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n01:128002] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n01:128002] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n01:128002] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n01:128002] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n12:178600] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n12:138735] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n12:138735] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n05:120329] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n05:120329] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n05:120329] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n05:120329] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n05:120329] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n17:123236] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n17:123236] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n17:123236] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n17:123236] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n17:123236] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n17:123236] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n17:123236] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n17:123236] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n17:123236] *** End of error message *** +[0x1000000504d8] +[b10n16:164552] [ 1] [0x3fffeaf13550] +[b10n16:164552] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n16:164552] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n16:164552] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n01:165122] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n01:165122] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n01:165122] [14] [b14n04:126857] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n11:160796] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n11:160796] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n11:160796] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n11:160796] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n07:124349] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n07:124349] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n07:124349] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n07:124349] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n07:124349] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n09:155760] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n10:16971] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n10:16971] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n10:16971] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n10:16971] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n10:16971] [10] [0x1000000504d8] +[b01n11:03983] [ 1] [0x3fffe1038190] +[b01n11:03983] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n11:03983] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n11:03983] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n07:160801] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n07:160801] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n07:160801] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n06:120732] *** End of error message *** +[b23n06:117529] *** Process received signal *** +[b23n06:117529] Signal: Segmentation fault (11) +[b23n06:117529] Signal code: (3) +[b23n06:117529] Failing at address: 0x14170000000000a0 +[b23n06:117529] [ 0] [0x1000000504d8] +[b23n06:117529] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n11:120584] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n11:120584] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n11:120584] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n11:120584] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n11:120584] *** End of error message *** +[a28n18:179166] *** Process received signal *** +[a28n18:179166] Signal: Segmentation fault (11) +[a28n18:179166] Signal code: (3) +[a28n18:179166] Failing at address: 0xb170000000000a0 +[a28n18:179166] [ 0] [0x1000000504d8] +[a28n18:179166] [ 1] [b17n13:123513] *** Process received signal *** +[b17n13:123513] Signal: Segmentation fault (11) +[b17n13:123513] Signal code: (3) +[b17n13:123513] Failing at address: 0x14170000000000a0 +[b17n13:123513] [ 0] [a31n04:158634] *** Process received signal *** +[a31n04:158634] Signal: Segmentation fault (11) +[a31n04:158634] Signal code: (3) +[a31n04:158634] Failing at address: 0xb170000000000a0 +[a31n04:158634] [ 0] [b13n11:134520] [ 0] [0x1000000504d8] +[b13n11:134520] [ 1] [0x3fffd1c870c0] +[b13n11:134520] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n11:134520] [ 3] [0x1000000504d8] +[a30n17:176411] [ 1] [0x3fffc54df900] +[a30n17:176411] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n17:176411] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n17:176411] [ 4] [b08n14:178323] *** Process received signal *** +[b08n14:178323] Signal: Segmentation fault (11) +[b08n14:178323] Signal code: (3) +[b08n14:178323] Failing at address: 0xb170000000000a0 +[b08n14:178323] [ 0] [0x1000000504d8] +[b08n14:178323] [ 1] [0x3fffd7d4d1d0] +[b08n14:178323] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n14:178323] [ 3] [a33n17:168153] [ 0] [0x1000000504d8] +[a33n17:168153] [ 1] [0x3fffe490fb90] +[a33n17:168153] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n17:168153] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n17:168153] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n04:44758] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n04:44758] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n04:44758] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n09:119632] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n09:119632] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n09:119632] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n03:154293] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n03:154293] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n03:154293] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n03:154293] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n13:171107] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n13:171107] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n13:171107] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n07:168961] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n07:168961] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n07:168961] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n07:168961] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n07:168961] [11] [a33n05:139437] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n01:163390] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n05:174779] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n05:174779] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n05:174779] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n05:174779] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n10:118389] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n01:96753] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n01:96753] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n16:154181] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n16:154181] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n16:154181] [ 8] [b03n03:06821] *** Process received signal *** +[b03n03:06821] Signal: Segmentation fault (11) +[b03n03:06821] Signal code: (3) +[b03n03:06821] Failing at address: 0x14170000000000a0 +[b03n03:06821] [ 0] [0x1000000504d8] +[b03n03:06821] [ 1] [0x3fffdfda3220] +[b03n03:06821] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n03:06821] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n03:06821] [ 4] [a36n05:139909] *** Process received signal *** +[a36n05:139909] Signal: Segmentation fault (11) +[a36n05:139909] Signal code: (3) +[a36n05:139909] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n13:123052] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n13:123052] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n13:123052] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n13:123052] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n13:123052] *** End of error message *** +[a30n10:169207] *** Process received signal *** +[a30n10:169207] Signal: Segmentation fault (11) +[a30n10:169207] Signal code: (3) +[a30n10:169207] Failing at address: 0x5170000000000a0 +[a30n10:169207] [ 0] [0x1000000504d8] +[a30n10:169207] [ 1] [b07n12:160959] *** Process received signal *** +[b07n12:160959] Signal: Segmentation fault (11) +[b07n12:160959] Signal code: (3) +[b07n12:160959] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n05:120228] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n05:120228] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n05:120228] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n05:120228] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n05:120228] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n13:154233] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n13:154233] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n13:154233] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n13:154233] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n13:154233] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n13:154233] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n13:154233] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n13:154233] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n13:154233] *** End of error message *** +[b07n03:172646] *** Process received signal *** +[b07n03:172646] Signal: Segmentation fault (11) +[b07n03:172646] Signal code: (3) +[b07n03:172646] Failing at address: 0xb170000000000a0 +[b07n03:172646] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n13:133627] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n13:133627] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n13:133627] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n13:133627] [12] [b03n01:170763] *** Process received signal *** +[b03n01:170763] Signal: Segmentation fault (11) +[b03n01:170763] Signal code: (3) +[b03n01:170763] Failing at address: 0xb170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n15:120130] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n15:120130] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n15:120130] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n15:120130] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n15:120130] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n14:152744] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n14:152744] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n14:152744] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n07:172879] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n07:172879] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n11:156152] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n11:156152] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n07:165088] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n07:165088] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n07:165088] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n07:165088] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n06:179454] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n06:179454] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n06:179454] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n10:140413] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n02:122176] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n02:122176] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n02:122176] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n18:128542] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n10:10926] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n10:10926] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n10:10926] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n10:10926] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n15:139750] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n15:139750] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n15:139750] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n15:139750] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n15:139750] *** End of error message *** +[0x3fffc97b15e0] +[b18n16:124958] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n16:124958] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n16:124958] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n16:124958] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n02:74430] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n08:127443] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n08:127443] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n08:127443] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n08:127443] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n08:127443] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n08:127443] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n18:10724] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n12:123561] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n12:123561] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n12:123561] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n12:123561] [14] [0x3ffffd01c120] +[b19n10:120975] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n10:120975] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n10:120975] [ 4] [b18n02:125371] *** Process received signal *** +[b18n02:125371] Signal: Segmentation fault (11) +[b18n02:125371] Signal code: (3) +[b18n02:125371] Failing at address: 0xb170000000000a0 +[b18n02:125371] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n05:176909] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n05:176909] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n05:176909] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n04:130744] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n04:130744] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n08:146118] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n08:146118] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n08:146118] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n08:146118] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n08:146118] [10] [0x1000000504d8] +[b09n09:170654] [ 1] [0x3fffe70cdaa0] +[b09n09:170654] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n09:170654] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n09:170654] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n12:122874] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n12:122874] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n12:122874] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n12:122874] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n03:135035] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n03:135035] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n03:135035] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n03:135035] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n03:135035] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n03:135035] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n07:156490] [ 6] [0x1000000504d8] +[b15n10:122759] [ 1] [0x3fffc0eb4a00] +[b15n10:122759] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n10:122759] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n10:122759] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n10:122759] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n14:126121] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n07:02839] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n07:02839] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n07:02839] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n07:02839] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n14:122440] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n14:122440] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n14:122440] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n14:122440] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n14:122440] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n14:122440] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n14:122440] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n14:122440] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n14:122440] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n11:134746] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n11:134746] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n11:134746] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n11:134746] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n05:173820] *** End of error message *** +[10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n04:136993] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n04:136993] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n04:136993] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n04:136993] [14] [b15n01:123573] *** Process received signal *** +[b15n01:123573] Signal: Segmentation fault (11) +[b15n01:123573] Signal code: (3) +[b15n01:123573] Failing at address: 0x14170000000000a0 +[b15n01:123573] [ 0] [0x1000000504d8] +[b15n01:123573] [ 1] [0x3fffd77ccc80] +[b15n01:123573] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n01:123573] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n12:145266] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n12:145266] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n12:145266] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n12:145266] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n12:145266] *** End of error message *** +[b15n03:119827] [ 0] [0x1000000504d8] +[b15n03:119827] [ 1] [0x3fffd76f4d10] +[b15n03:119827] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n03:119827] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n03:119827] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n03:119827] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n03:119827] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n03:119827] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n03:119827] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n03:119827] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n03:119827] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n03:119827] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n03:119827] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n03:119827] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n03:119827] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n03:119827] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n07:125447] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n08:165285] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n08:165285] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n08:165285] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n08:165285] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n08:165285] [14] [b17n08:124117] *** Process received signal *** +[b17n08:124117] Signal: Segmentation fault (11) +[b17n08:124117] Signal code: (3) +[b17n08:124117] Failing at address: 0x5170000000000a0 +[b17n08:124117] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n03:124681] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n04:177594] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n04:177594] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n04:177594] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n11:128956] [ 4] [b07n07:163251] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n09:143896] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n04:117352] *** End of error message *** +[0x3fffe82e1f20] +[b13n08:144852] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n08:144852] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n08:144852] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n07:156142] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n07:156142] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n07:156142] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n07:156142] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n18:122551] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n18:122551] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n18:122551] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n15:148163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n15:148163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n15:148163] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n15:148163] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n13:138088] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n13:138088] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n15:32399] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n15:32399] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n15:32399] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n15:32399] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n15:32399] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n15:32399] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n15:32399] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n17:168163] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n17:168163] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n17:168163] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n17:168163] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n17:168163] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n17:168163] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n17:168163] [10] [0x3fffd5100270] +[a32n12:164380] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n12:164380] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n12:164380] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n12:164380] [ 5] [b19n16:130374] *** Process received signal *** +[b19n16:130374] Signal: Segmentation fault (11) +[b19n16:130374] Signal code: (3) +[b19n16:130374] Failing at address: 0xb170000000000a0 +[b19n16:130374] [ 0] [0x1000000504d8] +[b19n16:130374] [ 1] [0x3fffcf66bb00] +[b19n16:130374] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n16:130374] [ 3] [b20n05:121824] *** Process received signal *** +[b20n05:121824] Signal: Segmentation fault (11) +[b20n05:121824] Signal code: (3) +[b20n05:121824] Failing at address: 0x14170000000000a0 +[b09n01:163022] *** Process received signal *** +[b09n01:163022] Signal: Segmentation fault (11) +[b09n01:163022] Signal code: (3) +[b09n01:163022] Failing at address: 0x14170000000000a0 +[b09n01:163022] [ 0] [0x1000000504d8] +[b09n01:163022] [ 1] [0x3ffff2ff8950] +[b09n01:163022] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n01:163022] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n04:168487] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n04:168487] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n04:168487] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n04:168487] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n04:168487] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n04:168487] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n04:168487] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n04:168487] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n04:168487] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n04:168487] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n04:168487] [14] [ 4] [a30n11:163257] *** Process received signal *** +[a30n11:163257] Signal: Segmentation fault (11) +[a30n11:163257] Signal code: (3) +[a30n11:163257] Failing at address: 0xb170000000000a0 +[a30n11:163257] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n16:147284] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n16:147284] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n16:147284] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n16:147284] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n16:147284] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n16:147284] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n16:147284] *** End of error message *** +[b17n17:125806] *** Process received signal *** +[b17n17:125806] Signal: Segmentation fault (11) +[b17n17:125806] Signal code: (3) +[b17n17:125806] Failing at address: 0x14170000000000a0 +[b17n17:125806] [ 0] [0x1000000504d8] +[b17n17:125806] [ 1] [0x3fffd8a889f0] +[b17n17:125806] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n17:125806] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n12:178717] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n12:178717] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n12:178717] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n13:117507] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n13:117507] *** End of error message *** +[b13n10:134731] *** Process received signal *** +[b13n10:134731] Signal: Segmentation fault (11) +[b13n10:134731] Signal code: (3) +[b13n10:134731] Failing at address: 0xb170000000000a0 +[b13n10:134731] [ 0] [0x1000000504d8] +[b13n10:134731] [ 1] [0x3ffff2966250] +[b13n10:134731] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n10:134731] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n14:119487] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n14:119487] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n14:119487] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n14:119487] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n14:150131] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n14:150131] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n14:150131] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n15:120060] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n04:126347] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n04:126347] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n04:126347] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n04:126347] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n04:126347] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n04:126347] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n13:168159] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n13:168159] [ 6] [0x3fffe95785e0] +[b06n13:166923] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n13:166923] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n13:166923] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n13:166923] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n09:161229] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n09:161229] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n11:153419] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n11:153419] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n03:158658] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n03:158658] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b22n02:119530] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n04:166205] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n16:128442] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n16:128442] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n16:128442] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n13:123883] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n13:123883] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n13:123883] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n13:123883] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n13:123883] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n16:173547] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n16:173547] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n16:173547] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n18:164800] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n18:164800] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n18:164800] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n18:164800] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n05:164941] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n05:164941] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n05:164941] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n05:164941] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n05:164941] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n05:164941] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a27n08:54624] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n08:54624] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n08:54624] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n18:08430] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n18:08430] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n18:08430] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n18:08430] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n18:131582] [ 4] [a33n02:167596] *** Process received signal *** +[a33n02:167596] Signal: Segmentation fault (11) +[a33n02:167596] Signal code: (3) +[a33n02:167596] Failing at address: 0x5170000000000a0 +[a33n02:167596] [ 0] [0x1000000504d8] +[a33n02:167596] [ 1] [0x3ffff84c8e80] +[a33n02:167596] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n02:167596] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n10:162608] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n10:162608] *** End of error message *** +[ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n12:120127] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n17:129930] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n17:129930] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n17:129930] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n12:132103] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n12:132103] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n12:132103] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n12:132103] *** End of error message *** +[0x1000000504d8] +[b19n15:124799] [ 1] [0x3fffd8aaf680] +[b19n15:124799] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n15:124799] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n15:124799] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n15:124799] [ 5] [b18n08:131376] *** Process received signal *** +[b18n08:131376] Signal: Segmentation fault (11) +[b18n08:131376] Signal code: (3) +[b18n08:131376] Failing at address: 0x14170000000000a0 +[b18n08:131376] [ 0] [0x1000000504d8] +[b18n08:131376] [ 1] [0x3fffe53dc450] +[b18n08:131376] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n08:131376] [ 3] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n06:167371] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n06:167371] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n06:167371] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n06:167371] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n06:167371] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n06:167371] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n06:167371] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n06:167371] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n06:167371] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n08:116175] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n08:116175] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n08:116175] *** End of error message *** +[b06n16:165924] *** Process received signal *** +[b06n16:165924] Signal: Segmentation fault (11) +[b06n16:165924] Signal code: (3) +[b06n16:165924] Failing at address: 0xb170000000000a0 +[b06n16:165924] [ 0] [0x1000000504d8] +[b06n16:165924] [ 1] [0x3ffffb091ae0] +[b06n16:165924] [ 2] [0x1000000504d8] +[b19n03:123618] [ 1] [0x3ffff40401f0] +[b19n03:123618] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n03:123618] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n03:123618] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n03:123618] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n03:123618] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n17:124164] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n06:127272] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n06:127272] *** End of error message *** +[b21n07:117362] *** Process received signal *** +[b21n07:117362] Signal: Segmentation fault (11) +[b21n07:117362] Signal code: (3) +[b21n07:117362] Failing at address: 0x14170000000000a0 +[b21n07:117362] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n07:133375] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n07:133375] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n02:176048] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n02:176048] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n02:176048] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n03:38500] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n03:38500] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n03:38500] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n03:38500] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n03:38500] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n03:38500] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n03:38500] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n03:38500] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n03:38500] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a29n16:157886] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a29n16:157886] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n11:160350] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n11:160350] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n11:160350] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n11:160350] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n13:114394] *** End of error message *** +[b20n01:128827] *** Process received signal *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n10:142933] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n10:142933] *** End of error message *** +[b21n16:116522] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n09:123415] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n09:123415] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n09:123415] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n09:123415] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n04:41668] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n04:41668] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n04:41668] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n04:41668] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n04:41668] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n04:41668] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n04:41668] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n04:41668] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n04:41668] *** End of error message *** +[0x1000000504d8] +[b03n02:06560] [ 1] [0x3ffffac1cc30] +[b03n02:06560] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n02:06560] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n02:06560] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n18:179696] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n18:179696] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n07:118164] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n01:48803] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n01:48803] *** End of error message *** +[b07n14:176066] *** Process received signal *** +[b07n14:176066] Signal: Segmentation fault (11) +[b07n14:176066] Signal code: (3) +[b07n14:176066] Failing at address: 0x14170000000000a0 +[b07n14:176066] [ 0] [0x1000000504d8] +[b07n14:176066] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n16:127349] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n16:127349] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n12:12763] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n12:167675] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n12:167675] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n12:167675] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n12:167675] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n12:167675] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n12:167675] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n07:180191] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n07:180191] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n07:180191] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n13:122029] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n13:122029] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n13:122029] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n13:122029] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n13:122029] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n13:122029] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n13:122029] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n13:122029] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n13:122029] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n13:122029] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n13:122029] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n12:122967] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n12:122967] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n12:122967] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n12:122967] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n14:128868] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n14:128868] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n17:03843] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n17:03843] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n17:03843] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n17:03843] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n17:149906] [11] [b19n06:126376] *** Process received signal *** +[b19n06:126376] Signal: Segmentation fault (11) +[b19n06:126376] Signal code: (3) +[b19n06:126376] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n05:135163] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n05:135163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n05:135163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n05:135163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n05:135163] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n08:08953] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n08:08953] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n18:126907] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n18:126907] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n18:126907] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n18:126907] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n14:169520] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n14:169520] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n06:143010] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n01:141815] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n01:141815] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n01:141815] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n01:141815] [ 8] [b03n06:01836] *** Process received signal *** +[b03n06:01836] Signal: Segmentation fault (11) +[b03n06:01836] Signal code: (3) +[b03n06:01836] Failing at address: 0x14170000000000a0 +[b03n06:01836] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n01:167300] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n01:167300] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n01:167300] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n13:176364] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n13:176364] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n13:176364] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n13:176364] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n13:176364] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n07:132086] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n10:176307] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n10:176307] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n10:176307] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n10:176307] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n10:176307] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n04:169909] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n17:150930] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n17:150930] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n17:150930] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n17:150930] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n02:164491] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n02:164491] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n02:164491] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n02:164491] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n02:164491] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n07:138204] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n07:138204] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n07:138204] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n07:138204] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n11:137870] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n11:137870] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n11:137870] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n11:137870] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n01:128155] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n03:10289] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n07:131670] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n07:131670] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n07:131670] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n07:131670] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n07:131670] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n16:01570] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n16:01570] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n16:01570] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n16:01570] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n02:166743] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n02:166743] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n02:166743] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n02:166743] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n02:166743] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n02:166743] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n02:166743] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n09:121751] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n09:121751] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n09:121751] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n09:121751] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n09:121751] *** End of error message *** +[a28n11:01173] *** Process received signal *** +[a28n11:01173] Signal: Segmentation fault (11) +[a28n11:01173] Signal code: (3) +[a28n11:01173] Failing at address: 0x14170000000000a0 +[a28n11:01173] [ 0] [0x1000000504d8] +[a28n11:01173] [ 1] [0x3fffccfe5e70] +[a28n11:01173] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n11:01173] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n13:123706] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n13:123706] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n13:123706] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n13:123706] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n13:123706] [10] [b11n11:152984] *** Process received signal *** +[b11n11:152984] Signal: Segmentation fault (11) +[b11n11:152984] Signal code: (3) +[b11n11:152984] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n11:171961] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n11:171961] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n11:171961] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n11:171961] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n14:161619] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n14:161619] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n14:161619] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n14:161619] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n14:161619] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n14:161619] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n14:161619] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n14:161619] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n09:173753] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n09:173753] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n09:173753] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n09:173753] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n09:173753] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n16:122491] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n16:122491] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n16:122491] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n12:148378] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n12:148378] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n12:148378] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n12:148378] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n02:143965] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n02:143965] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n02:143965] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n12:156915] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n12:156915] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n12:156915] [14] [a30n16:153288] [ 0] [0x1000000504d8] +[a30n16:153288] [ 1] [0x3fffc8aca200] +[a30n16:153288] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n16:153288] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n12:11142] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n12:11142] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n12:11142] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n17:119579] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n17:119579] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n17:119579] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n17:119579] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n17:119579] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n04:175996] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n04:175996] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n04:175996] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n04:175996] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b12n01:145730] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b12n01:145730] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b12n01:145730] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n01:145730] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n15:148583] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n15:148583] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n15:148583] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n15:148583] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n02:163318] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n02:163318] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n09:123810] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n17:164563] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n17:164563] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n12:115639] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n12:115639] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n12:115639] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n12:115639] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n12:115639] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n08:156368] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n08:156368] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n07:127357] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n08:121685] *** Process received signal *** +[b19n08:121685] Signal: Segmentation fault (11) +[b19n08:121685] Signal code: (3) +[b19n08:121685] Failing at address: 0x14170000000000a0 +[b19n08:121685] [ 0] [0x1000000504d8] +[b19n08:121685] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n15:137005] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n15:137005] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n15:137005] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n15:137005] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n15:137005] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n13:163195] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n13:165258] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n12:177117] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n12:177117] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n12:177117] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n12:177117] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n12:177117] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n12:177117] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n12:177117] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n12:177117] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n12:177117] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n12:177117] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n12:177117] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n12:177117] *** End of error message *** +[a30n08:163097] [ 0] [0x1000000504d8] +[a30n08:163097] [ 1] [0x3fffe055ad80] +[a30n08:163097] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n08:163097] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n08:163097] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n08:163097] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n08:163097] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n08:163097] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n08:163097] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n08:163097] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n08:163097] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n11:02683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n11:02683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n11:02683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n11:02683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n11:02683] *** End of error message *** +[0x1000000504d8] +[a35n17:146797] [ 1] [0x3fffef25f3d0] +[a35n17:146797] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n17:146797] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n17:146797] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n17:146797] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n03:168933] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n11:120239] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n11:120239] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n11:120239] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n11:120239] [12] [a35n02:149231] *** Process received signal *** +[a35n02:149231] Signal: Segmentation fault (11) +[a35n02:149231] Signal code: (3) +[a35n02:149231] Failing at address: 0x14170000000000a0 +[a35n02:149231] [ 0] [0x1000000504d8] +[a35n02:149231] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n11:175593] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n11:175593] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n11:175593] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n11:175593] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n11:175593] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n11:175593] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n11:175593] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n11:175593] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n15:123038] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n15:123038] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n15:123038] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n15:123038] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n15:123038] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n15:123038] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n15:123038] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n15:123038] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n15:123038] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n06:177007] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n06:177007] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n06:177007] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n06:177007] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n14:163549] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n06:10223] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n06:10223] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n01:138007] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n01:138007] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n01:138007] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n03:135162] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n02:178476] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n02:178476] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n03:136664] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n03:136664] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n03:136664] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n03:136664] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a26n01:65724] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n01:65724] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n01:65724] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n01:65724] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n03:125242] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n03:125242] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n03:125242] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n15:161753] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n15:161753] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n06:154620] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n06:154620] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n06:154620] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n06:154620] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n06:154620] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n04:169702] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n04:169702] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n04:169702] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n04:169702] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n05:162568] [ 6] [b01n03:01355] [ 0] [0x1000000504d8] +[b01n03:01355] [ 1] [0x3fffcea64c70] +[b01n03:01355] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n03:01355] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n02:122558] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n02:122558] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n11:148671] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n11:148671] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n11:148671] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n17:119465] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n17:119465] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n17:119465] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n17:119465] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n09:147793] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n09:147793] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n09:147793] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n06:133955] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n06:133955] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n05:135119] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n05:135119] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n05:135119] [14] [0x3fffc6a5ada0] +[b19n02:123703] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n02:123703] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n02:123703] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n03:151882] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n03:151882] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n03:151882] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n01:164961] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n01:164961] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n01:164961] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n01:164961] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n06:130540] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n06:130540] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n06:130540] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n06:130540] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n06:130540] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n17:163816] [ 9] [b19n05:131262] *** Process received signal *** +[b19n05:131262] Signal: Segmentation fault (11) +[b19n05:131262] Signal code: (3) +[b19n05:131262] Failing at address: 0x14170000000000a0 +[b19n05:131262] [ 0] [0x1000000504d8] +[b19n05:131262] [ 1] [0x3fffcd60ab40] +[b19n05:131262] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n17:118572] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n17:118572] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n17:118572] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n17:118572] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n01:141343] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n01:141343] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n01:141343] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n01:141343] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n01:179259] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n01:179259] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n02:147882] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n02:147882] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n02:147882] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n02:147882] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n03:130348] [ 4] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n03:153853] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n03:153853] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n03:153853] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n03:153853] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n03:153853] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n03:153853] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n08:164749] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n08:164749] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n08:164749] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n08:164749] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n08:164749] *** End of error message *** +[b03n04:04847] *** Process received signal *** +[b03n04:04847] Signal: Segmentation fault (11) +[b03n04:04847] Signal code: (3) +[b03n04:04847] Failing at address: 0x14170000000000a0 +[b03n04:04847] [ 0] [0x1000000504d8] +[b03n04:04847] [ 1] [0x3fffebd5e890] +[b03n04:04847] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n04:04847] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n18:118683] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n18:118683] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n18:118683] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n18:118683] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n18:118683] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n18:118683] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n18:118683] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n18:118683] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n18:118683] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n06:163674] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n06:162916] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n06:162916] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n06:162916] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n06:162916] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n06:162916] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n06:171500] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n06:171500] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n06:171500] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n06:171500] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n06:171500] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n17:123237] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n17:123237] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n17:123237] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n17:123237] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n17:123237] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n01:165122] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n01:128001] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n01:128001] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n12:138734] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n12:138734] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n12:138734] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n12:138734] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n12:138734] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n16:164552] [ 5] [b15n06:120730] *** Process received signal *** +[b15n06:120730] Signal: Segmentation fault (11) +[b15n06:120730] Signal code: (3) +[b15n06:120730] Failing at address: 0xb170000000000a0 +[b15n06:120730] [ 0] [0x1000000504d8] +[b15n06:120730] [ 1] [0x3ffff8e05bc0] +[b15n06:120730] [ 2] [b15n05:120330] [ 1] [0x3fffefbd4780] +[b15n05:120330] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n05:120330] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n05:120330] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n11:160796] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n11:160796] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n11:160796] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n11:160796] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n11:160796] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n07:124348] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n09:155759] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n09:155759] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n09:155759] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n10:16970] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n10:16970] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n10:16970] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n10:16970] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n10:16970] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n11:03982] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n11:03982] [ 5] [ 0] [0x1000000504d8] +[b01n12:178602] [ 1] [0x3ffffcf2e070] +[b01n12:178602] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n12:178602] [ 3] [0x3fffd398cf20] +[b23n06:117529] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b23n06:117529] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n18:179167] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n18:179167] [ 5] [a31n04:158635] *** Process received signal *** +[a31n04:158635] Signal: Segmentation fault (11) +[a31n04:158635] Signal code: (3) +[a31n04:158635] Failing at address: 0xb170000000000a0 +[a31n04:158635] [ 0] [0x1000000504d8] +[a31n04:158635] [ 1] [0x3ffff5bce8b0] +[a31n04:158635] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n04:158635] [ 3] [b13n11:134521] *** Process received signal *** +[b13n11:134521] Signal: Segmentation fault (11) +[b13n11:134521] Signal code: (3) +[b13n11:134521] Failing at address: 0x14170000000000a0 +[b13n11:134521] [ 0] [0x1000000504d8] +[b13n11:134521] [ 1] [0x3fffe3a7f120] +[b13n11:134521] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n11:134521] [ 3] [0x1000000504d8] +[b17n13:123513] [ 1] [0x3ffffa1d6df0] +[b17n13:123513] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n13:123513] [ 3] [0x1000000504d8] +[b08n14:178322] [ 1] [0x3fffe8fa62b0] +[b08n14:178322] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b08n14:178322] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n14:178322] [ 4] [a33n17:168152] *** Process received signal *** +[a33n17:168152] Signal: Segmentation fault (11) +[a33n17:168152] Signal code: (3) +[a33n17:168152] Failing at address: 0x14170000000000a0 +[a33n17:168152] [ 0] [0x1000000504d8] +[a33n17:168152] [ 1] [0x3fffe97afb30] +[a33n17:168152] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a33n17:168152] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n17:168152] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n17:176411] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n04:44759] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n04:44759] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n04:44759] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n04:44759] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n03:154292] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n03:154292] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n09:119631] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n09:119631] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n09:119631] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n09:119631] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n09:119631] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n07:168960] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n07:168960] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n07:168960] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n13:171106] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n13:171106] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n13:171106] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n13:171106] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n13:171106] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n05:174778] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n05:174778] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n05:174778] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n05:174778] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n05:174778] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n05:174778] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n01:163390] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n10:118389] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n10:118389] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n01:96752] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n01:96752] [ 5] [b03n01:170763] [ 0] [0x1000000504d8] +[b03n01:170763] [ 1] [0x3ffffd08c8e0] +[b03n01:170763] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n01:170763] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n01:170763] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n03:06822] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n03:06822] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n03:06822] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n03:06822] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n03:06822] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n03:06822] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n03:06822] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n03:06822] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n03:06822] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n03:06822] [14] [0x1000000504d8] +[a30n10:169206] [ 1] [0x3fffecf66c70] +[a30n10:169206] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n10:169206] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n10:169206] [ 4] [a36n05:139909] [ 0] [0x1000000504d8] +[a36n05:139909] [ 1] [0x3ffffd662ac0] +[a36n05:139909] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a36n05:139909] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n05:120227] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n05:120227] *** End of error message *** +[b07n12:160960] *** Process received signal *** +[b07n12:160960] Signal: Segmentation fault (11) +[b07n12:160960] Signal code: (3) +[b07n12:160960] Failing at address: 0x14170000000000a0 +[b07n12:160960] [ 0] [0x1000000504d8] +[b07n12:160960] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n13:154234] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n13:154234] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n13:154234] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n13:154234] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n13:154234] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n07:165087] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n07:165087] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n07:165087] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n07:165087] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n11:156151] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n11:156151] [10] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n14:152747] [13] [b07n03:172647] *** Process received signal *** +[b07n03:172647] Signal: Segmentation fault (11) +[b07n03:172647] Signal code: (3) +[b07n03:172647] Failing at address: 0xb170000000000a0 +[b07n03:172647] [ 0] [0x1000000504d8] +[b07n03:172647] [ 1] [0x3fffeabc8980] +[b07n03:172647] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n03:172647] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n03:172647] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n03:172647] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n03:172647] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n03:172647] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n03:172647] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n03:172647] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n03:172647] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n03:172647] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n03:172647] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n03:172647] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n03:172647] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n03:172647] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n16:154182] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n16:154182] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n13:133626] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n13:133626] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n13:133626] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n13:133626] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n13:133626] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n12:123561] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n06:179454] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n15:120129] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n15:120129] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n15:120129] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n15:120129] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n02:122177] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n02:122177] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n02:122177] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n02:122177] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n18:128542] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n18:128542] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n18:128542] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n10:10925] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n10:10925] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n10:10925] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n10:10925] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n10:10925] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n16:124957] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n16:124957] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n02:74430] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n08:127442] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n08:127442] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n08:127442] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n08:127442] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n08:127442] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n18:10723] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n18:10723] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n07:172880] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n07:172880] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n07:172880] [12] [b18n02:125372] *** Process received signal *** +[b18n02:125372] Signal: Segmentation fault (11) +[b18n02:125372] Signal code: (3) +[b18n02:125372] Failing at address: 0xb170000000000a0 +[b18n02:125372] [ 0] [0x1000000504d8] +[b18n02:125372] [ 1] [0x3fffc486fed0] +[b18n02:125372] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n02:125372] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n05:176909] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n05:176909] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n08:146117] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n08:146117] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n08:146117] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n04:130745] *** End of error message *** +[0x3fffe8a731d0] +[b19n10:120974] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n10:120974] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n10:120974] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n10:120974] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n10:120974] [ 6] [0x1000000504d8] +[b09n09:170655] [ 1] [0x3fffc1ed62c0] +[b09n09:170655] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n09:170655] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n09:170655] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n09:170655] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n12:122873] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n12:122873] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n12:122873] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n12:122873] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n12:122873] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n07:156489] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n07:156489] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n03:135034] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n03:135034] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n03:135034] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n03:135034] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n03:135034] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n14:122441] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n14:122441] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n14:122441] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n10:122760] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n10:122760] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n10:122760] [ 7] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n07:02839] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n07:02839] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n11:134745] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n11:134745] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n11:134745] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n11:134745] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n11:134745] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n14:126121] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n14:126121] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n14:126121] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n04:136994] *** End of error message *** +[0x1000000504d8] +[b15n01:123572] [ 1] [0x3fffdbf0dbe0] +[b15n01:123572] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n01:123572] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n01:123572] [ 4] [b15n03:119828] *** Process received signal *** +[b15n03:119828] Signal: Segmentation fault (11) +[b15n03:119828] Signal code: (3) +[b15n03:119828] Failing at address: 0x14170000000000a0 +[b15n03:119828] [ 0] [0x1000000504d8] +[b15n03:119828] [ 1] [0x3fffd8ae3bf0] +[b15n03:119828] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n03:119828] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n03:119828] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n03:119828] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n03:119828] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n03:119828] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n03:119828] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n03:119828] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n03:119828] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n03:119828] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n03:119828] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n03:119828] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n03:119828] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n03:119828] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n07:125448] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n07:125448] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n08:165284] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n08:165284] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n08:165284] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n08:165284] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n08:165284] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n08:165284] [12] [0x1000000504d8] +[b17n08:124117] [ 1] [0x3ffffc9c6e40] +[b17n08:124117] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n08:124117] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n08:124117] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n03:124681] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n04:177595] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n04:177595] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n04:177595] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n04:177595] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n04:177595] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n11:128957] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n11:128957] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n11:128957] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n07:163250] [ 5] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n04:117351] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n04:117351] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n08:144851] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n08:144851] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n15:148164] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n15:148164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n15:148164] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n15:148164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n07:156143] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n07:156143] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n07:156143] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n07:156143] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n07:156143] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n18:122552] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n18:122552] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n18:122552] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n14:150130] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n14:150130] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n14:150130] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n14:150130] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n15:32400] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n15:32400] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n15:32400] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n15:32400] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n15:32400] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n15:32400] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n12:164380] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n12:164380] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n12:164380] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n12:164380] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n12:164380] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n12:164380] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n12:164380] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n12:164380] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n12:164380] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n12:164380] *** End of error message *** +[0x1000000504d8] +[b19n16:130373] [ 1] [0x3fffe3d3fa40] +[b19n16:130373] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n16:130373] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n16:130373] [ 4] [b20n05:121825] *** Process received signal *** +[b20n05:121825] Signal: Segmentation fault (11) +[b20n05:121825] Signal code: (3) +[b20n05:121825] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n01:163021] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n13:168158] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n13:168158] *** End of error message *** +[0x1000000504d8] +[a30n11:163258] [ 1] [0x3fffd64aa9d0] +[a30n11:163258] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n11:163258] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n11:163258] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n15:151249] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n12:178716] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n12:178716] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n12:178716] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n12:178716] [12] [0x1000000504d8] +[b13n10:134730] [ 1] [0x3fffd50ed8f0] +[b13n10:134730] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b13n10:134730] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n10:134730] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n17:168162] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n17:168162] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n17:168162] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n17:168162] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n14:119486] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n14:119486] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n13:138087] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n13:138087] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n13:138087] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n13:138087] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n13:138087] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n04:126348] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n04:126348] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n04:126348] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n04:126348] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n04:168488] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n04:168488] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n15:120060] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n15:120060] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n15:120060] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n17:125805] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n13:166922] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n13:166922] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n13:166922] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n13:166922] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n13:166922] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n03:158659] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n03:158659] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n03:158659] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n03:158659] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n03:158659] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n11:153419] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n11:153419] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n11:153419] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n11:153419] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n11:153419] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n11:153419] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n11:153419] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n11:153419] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n11:153419] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n16:128441] [ 4] [0x3fffde6defc0] +[b19n15:124798] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n15:124798] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n15:124798] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n15:124798] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n15:124798] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n02:119531] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n02:119531] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n02:119531] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n02:119531] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n13:123884] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n13:123884] *** End of error message *** +[0x3fffce3ddff0] +[b09n16:173546] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n16:173546] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n16:173546] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n18:164799] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n18:164799] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n18:164799] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a27n08:54623] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a27n08:54623] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a27n08:54623] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n08:54623] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n05:164942] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n05:164942] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n05:164942] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n18:08431] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n18:08431] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n18:08431] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n18:08431] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n18:08431] *** End of error message *** +[b14n18:131581] *** Process received signal *** +[b14n18:131581] Signal: Segmentation fault (11) +[b14n18:131581] Signal code: (3) +[b14n18:131581] Failing at address: 0x14170000000000a0 +[b14n18:131581] [ 0] [0x1000000504d8] +[b14n18:131581] [ 1] [0x3fffc014e400] +[b14n18:131581] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n18:131581] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n02:167595] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n02:167595] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n12:132102] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n04:166205] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n12:120127] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n12:120127] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n12:120127] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n12:120127] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n17:129930] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n17:129930] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n08:131376] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n08:131376] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n06:167370] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n14:133063] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n14:133063] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n14:133063] [ 9] [0x3ffff9d82780] +[b06n16:165923] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n16:165923] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n16:165923] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n03:123617] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n03:123617] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n03:123617] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n03:123617] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n03:123617] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n03:123617] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n03:123617] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n06:127273] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n06:127273] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n06:127273] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n06:127273] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n06:127273] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n06:127273] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n06:127273] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n06:127273] *** End of error message *** +[b21n07:117361] *** Process received signal *** +[b21n07:117361] Signal: Segmentation fault (11) +[b21n07:117361] Signal code: (3) +[b21n07:117361] Failing at address: 0x14170000000000a0 +[b21n07:117361] [ 0] [0x1000000504d8] +[b21n07:117361] [ 1] [0x3ffffdc613e0] +[b21n07:117361] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n07:117361] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n07:133376] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n07:133376] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n07:133376] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n07:133376] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n07:133376] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n07:133376] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n07:133376] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n16:157887] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n16:157887] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n16:157887] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n03:38499] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n03:38499] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n03:38499] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n03:38499] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n03:38499] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n03:38499] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n03:38499] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n02:176047] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n02:176047] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n02:176047] *** End of error message *** +[b20n01:128828] *** Process received signal *** +[b20n01:128828] Signal: Segmentation fault (11) +[b20n01:128828] Signal code: (3) +[b20n01:128828] Failing at address: 0xb170000000000a0 +[b20n01:128828] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n11:160349] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n11:160349] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n11:160349] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n11:160349] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n11:160349] *** End of error message *** +[b19n06:126376] [ 0] [0x1000000504d8] +[b19n06:126376] [ 1] [0x3fffc2829080] +[b19n06:126376] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n06:126376] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n04:41669] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n04:41669] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n04:41669] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n04:41669] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n04:41669] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n18:179697] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n18:179697] *** End of error message *** +[0x1000000504d8] +[b03n02:06559] [ 1] [0x3ffff08f1600] +[b03n02:06559] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n02:06559] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n02:06559] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n02:06559] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n07:118164] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n14:176067] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n16:127350] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n16:127350] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n16:127350] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n16:127350] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n16:127350] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n12:12762] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n12:12762] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n07:180191] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n07:180191] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n12:122966] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n12:122966] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n12:122966] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n14:128867] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n14:128867] *** End of error message *** +[b02n17:03842] *** Process received signal *** +[b02n17:03842] Signal: Segmentation fault (11) +[b02n17:03842] Signal code: (3) +[b02n17:03842] Failing at address: 0xb170000000000a0 +[b02n17:03842] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n13:176363] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n13:176363] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n13:176363] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n13:176363] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n13:176363] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n09:123416] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n09:123416] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n09:123416] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n09:123416] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n05:135162] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n05:135162] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n05:135162] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n05:135162] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b05n08:08952] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n18:126906] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n18:126906] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n18:126906] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n01:141814] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n01:141814] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n01:141814] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n01:141814] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n06:143009] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n06:143009] *** End of error message *** +[b03n06:01837] *** Process received signal *** +[b03n06:01837] Signal: Segmentation fault (11) +[b03n06:01837] Signal code: (3) +[b03n06:01837] Failing at address: 0x14170000000000a0 +[b03n06:01837] [ 0] [0x1000000504d8] +[b03n06:01837] [ 1] [0x3ffffd969290] +[b03n06:01837] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n06:01837] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n06:01837] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n07:132087] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n07:132087] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n07:132087] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n07:132087] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n07:132087] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n01:167301] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n01:167301] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n01:167301] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n01:167301] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n01:167301] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n10:176306] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n10:176306] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n10:176306] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n10:176306] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n17:150931] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n17:150931] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n17:150931] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n17:150931] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n17:150931] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n17:150931] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n04:169910] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n04:169910] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n04:169910] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n17:149906] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n17:149906] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n02:164492] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n02:164492] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n02:164492] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n02:164492] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n02:164492] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n07:138205] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n07:138205] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n07:138205] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n07:138205] [13] [b18n01:128156] *** Process received signal *** +[b18n01:128156] Signal: Segmentation fault (11) +[b18n01:128156] Signal code: (3) +[b18n01:128156] Failing at address: 0x14170000000000a0 +[b18n01:128156] [ 0] [0x1000000504d8] +[b18n01:128156] [ 1] [0x3fffe4acf440] +[b18n01:128156] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n01:128156] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n07:131669] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n07:131669] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n07:131669] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n07:131669] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n07:131669] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n11:137869] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n11:137869] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n11:137869] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n11:137869] [13] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n16:01569] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n16:01569] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n16:01569] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n16:01569] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n16:01569] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n16:01569] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n16:01569] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n09:121752] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n09:121752] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n14:161620] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n11:01174] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n13:123705] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n13:123705] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n13:123705] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n13:123705] [13] [b11n11:152984] [ 0] [0x1000000504d8] +[b11n11:152984] [ 1] [0x3fffe35f0e20] +[b11n11:152984] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b11n11:152984] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b11n11:152984] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n11:171962] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n11:171962] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n09:173752] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n09:173752] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n16:122492] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n16:122492] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n16:122492] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n16:122492] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n16:122492] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n17:164563] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n17:164563] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n17:164563] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n17:164563] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n12:148378] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n16:153288] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n16:153288] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n16:153288] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n16:153288] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n16:153288] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n16:153288] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n16:153288] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n16:153288] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n12:11143] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n12:11143] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n12:11143] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n02:143965] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n04:175997] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n04:175997] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n04:175997] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n04:175997] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n04:175997] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n01:145730] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n01:145730] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n01:145730] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n01:145730] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n02:163319] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n02:163319] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n02:163319] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n02:163319] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n15:148582] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n15:148582] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n15:148582] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n15:148582] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n15:148582] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n09:123810] [ 7] [b23n12:115640] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n08:156369] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n08:156369] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n08:156369] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n12:156914] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n12:156914] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n12:156914] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n12:156914] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n12:156914] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n12:156914] *** End of error message *** +[b14n07:127357] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n13:163195] [11] [0x3fffd4468090] +[b19n08:121685] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n08:121685] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n08:121685] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n15:137004] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n15:137004] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n15:137004] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n15:137004] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n08:163096] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n08:163096] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n08:163096] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n08:163096] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n08:163096] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n11:02682] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n17:146796] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n17:146796] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n13:165258] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n13:165258] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n13:165258] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n03:168933] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n03:168933] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n11:120238] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n11:120238] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n11:120238] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n02:149230] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n02:149230] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n02:149230] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n01:138006] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n01:138006] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n01:138006] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n01:138006] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n06:177006] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n06:177006] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n11:175594] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n11:175594] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n11:175594] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n11:175594] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n11:175594] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n11:175594] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n11:175594] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n11:175594] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n11:175594] *** End of error message *** +[a30n14:163546] *** Process received signal *** +[a30n14:163546] Signal: Segmentation fault (11) +[a30n14:163546] Signal code: (3) +[a30n14:163546] Failing at address: 0xb170000000000a0 +[a30n14:163546] [ 0] [0x1000000504d8] +[a30n14:163546] [ 1] [0x3fffd2c89ce0] +[a30n14:163546] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n14:163546] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n14:163546] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n03:135162] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n06:10222] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n06:10222] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n03:136665] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n03:136665] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n03:136665] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n03:136665] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n02:178477] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n02:178477] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n02:178477] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n02:178477] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a26n01:65725] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a26n01:65725] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a26n01:65725] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n03:125243] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n03:125243] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n03:125243] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n03:125243] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n06:154619] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n06:154619] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n06:154619] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n06:154619] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n05:162569] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n05:162569] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n05:162569] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n05:162569] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n15:161753] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n15:161753] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n15:161753] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n04:169702] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n04:169702] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n04:169702] [10] [b01n03:01356] *** Process received signal *** +[b01n03:01356] Signal: Segmentation fault (11) +[b01n03:01356] Signal code: (3) +[b01n03:01356] Failing at address: 0xb170000000000a0 +[b01n03:01356] [ 0] [0x1000000504d8] +[b01n03:01356] [ 1] [0x3ffffb6949c0] +[b01n03:01356] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n03:01356] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n02:122559] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n02:122559] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n02:122559] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n02:122559] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n02:122559] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n11:148672] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n09:147794] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n17:119464] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n17:119464] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n17:119464] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n06:133956] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n06:133956] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n06:133956] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n06:133956] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n06:133956] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n06:133956] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n06:133956] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n06:133956] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n06:133956] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n05:135119] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n02:123703] [ 5] [a32n03:151883] *** Process received signal *** +[a32n03:151883] Signal: Segmentation fault (11) +[a32n03:151883] Signal code: (3) +[a32n03:151883] Failing at address: 0x5170000000000a0 +[a32n03:151883] [ 0] [0x1000000504d8] +[a32n03:151883] [ 1] [0x3fffe09d0730] +[a32n03:151883] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a32n03:151883] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a32n03:151883] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n03:151883] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n01:164960] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n01:164960] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n01:164960] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n06:130541] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n17:163816] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n17:118571] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n17:118571] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n17:118571] [11] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n05:131262] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n05:131262] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n05:131262] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n01:141344] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n01:141344] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n01:141344] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n01:141344] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n01:179260] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n01:179260] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n01:179260] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n03:130348] [ 5] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n02:147881] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n02:147881] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n03:153852] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n03:153852] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n03:153852] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n03:153852] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n03:153852] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n08:164748] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n08:164748] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n04:04846] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n04:04846] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n18:118684] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n18:118684] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n18:118684] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n18:118684] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n18:118684] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n18:118684] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n06:162915] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n06:171501] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n06:171501] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n06:171501] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n06:171501] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n06:171501] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n01:165123] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n01:165123] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n01:165123] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n01:165123] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n01:165123] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n01:128002] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n01:128002] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n01:128002] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n01:128002] [14] [0x1000000504d8] +[b10n16:164553] [ 1] [0x3fffda9ad0a0] +[b10n16:164553] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b10n16:164553] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b10n16:164553] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b10n16:164553] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n05:120329] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n05:120329] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n05:120329] [14] [b15n06:120731] *** Process received signal *** +[b15n06:120731] Signal: Segmentation fault (11) +[b15n06:120731] Signal code: (3) +[b15n06:120731] Failing at address: 0xb170000000000a0 +[b15n06:120731] [ 0] [0x1000000504d8] +[b15n06:120731] [ 1] [0x3fffd02bd780] +[b15n06:120731] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n06:120731] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n11:160795] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n11:160795] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n11:160795] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n11:160795] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n10:16971] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n10:16971] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n10:16971] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n10:16971] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n10:16971] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n07:124349] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n07:124349] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n07:124349] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n07:124349] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n07:124349] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n11:03983] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n11:03983] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n12:178602] [ 4] [0x3fffccfb8b60] +[a28n18:179166] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a28n18:179166] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n18:179166] [ 4] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n13:123513] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n11:134520] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n11:134520] [ 5] [0x1000000504d8] +[a31n04:158634] [ 1] [0x3fffd1aab160] +[a31n04:158634] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a31n04:158634] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n04:158634] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b08n14:178323] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n14:178323] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n04:44759] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n04:44759] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n04:44759] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n04:44759] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n04:44759] *** End of error message *** +[0x3ffff0bc6810] +[a30n10:169207] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n10:169207] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n10:169207] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n17:176411] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n03:154293] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n03:154293] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n03:154293] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n03:154293] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n03:154293] *** End of error message *** +[14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n09:119631] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n13:171107] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n13:171107] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n01:163390] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n07:168961] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n07:168961] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n07:168961] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n07:168961] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n05:174778] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n10:118389] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n01:96753] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n01:96753] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n01:96753] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n01:96753] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n12:123562] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n12:123562] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n12:123562] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n12:123562] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n12:123562] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n12:123562] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a36n05:139909] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n03:06821] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n03:06821] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n03:06821] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n17:168153] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n17:168153] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n17:168153] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n17:168153] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n18:128542] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n16:124957] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n16:124957] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n07:165088] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n07:165088] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n07:165088] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n07:165088] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n07:165088] [13] [0x1000000504d8] +[b07n03:172646] [ 1] [0x3ffffd82a360] +[b07n03:172646] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n03:172646] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n03:172646] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n03:172646] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n03:172646] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n03:172646] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n03:172646] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n03:172646] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n03:172646] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n03:172646] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n03:172646] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n03:172646] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n03:172646] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n03:172646] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a32n14:152745] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n14:152745] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n14:152745] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n14:152745] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n14:152745] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n13:133627] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n13:133627] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n13:133627] *** End of error message *** +[b03n01:170764] *** Process received signal *** +[b03n01:170764] Signal: Segmentation fault (11) +[b03n01:170764] Signal code: (3) +[b03n01:170764] Failing at address: 0xb170000000000a0 +[b03n01:170764] [ 0] [0x1000000504d8] +[b03n01:170764] [ 1] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n15:120130] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n15:120130] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n16:154181] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n16:154181] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n06:179454] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n02:122176] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n02:122176] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n02:122176] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n02:122176] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n10:10926] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n10:10926] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n10:10926] *** End of error message *** +[b07n12:160959] [ 0] [0x1000000504d8] +[b07n12:160959] [ 1] [0x3fffe8b54870] +[b07n12:160959] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n12:160959] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n12:160959] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n11:156152] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n11:156152] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n11:156152] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n11:156152] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n07:172880] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n07:172880] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n07:172880] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n08:127443] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n08:127443] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n08:127443] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n08:127443] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n18:10723] [ 9] [b07n05:176908] *** Process received signal *** +[b07n05:176908] Signal: Segmentation fault (11) +[b07n05:176908] Signal code: (3) +[b07n05:176908] Failing at address: 0xb170000000000a0 +[b07n05:176908] [ 0] [0x1000000504d8] +[b07n05:176908] [ 1] [0x1000000504d8] +[b18n02:125371] [ 1] [0x3ffffeb91e90] +[b18n02:125371] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n02:125371] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n02:125371] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n08:146117] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n08:146117] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n08:146117] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n08:146117] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n09:170654] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n09:170654] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n10:120975] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n10:120975] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n10:120975] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n10:120975] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n10:120975] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n12:122874] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n12:122874] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n12:122874] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n12:122874] [14] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n03:135035] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n03:135035] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n03:135035] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n03:135035] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n03:135035] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n10:122759] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n10:122759] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n10:122759] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n10:122759] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n10:122759] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n11:134746] *** End of error message *** +[b04n07:02837] *** Process received signal *** +[b04n07:02837] Signal: Segmentation fault (11) +[b04n07:02837] Signal code: (3) +[b04n07:02837] Failing at address: 0x5170000000000a0 +[b04n07:02837] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n14:126121] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n14:126121] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n04:136993] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n01:123573] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n01:123573] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n01:123573] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n07:125447] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n07:125447] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n08:165285] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n04:177594] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n08:124117] [ 5] [b20n03:124678] *** Process received signal *** +[b20n03:124678] Signal: Segmentation fault (11) +[b20n03:124678] Signal code: (3) +[b20n03:124678] Failing at address: 0xb170000000000a0 +[b20n03:124678] [ 0] [0x1000000504d8] +[b20n03:124678] [ 1] [0x3fffdc8da5c0] +[b20n03:124678] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n03:124678] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n03:124678] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n03:124678] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n03:124678] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n03:124678] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n03:124678] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n03:124678] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n03:124678] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n03:124678] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n03:124678] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n03:124678] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n11:128956] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n11:128956] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n07:163251] Signal: Segmentation fault (11) +[b07n07:163251] Signal code: (3) +[b07n07:163251] Failing at address: 0xb170000000000a0 +[b07n07:163251] [ 0] [0x1000000504d8] +[b07n07:163251] [ 1] [0x3fffd6bda0f0] +[b07n07:163251] [ 2] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n09:143896] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n08:144852] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n15:148163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n15:148163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n15:148163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n15:148163] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n07:156142] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n07:156142] *** End of error message *** +[11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n13:138087] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n15:32398] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n12:164379] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n16:130374] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n16:130374] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n01:163021] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n01:163021] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n01:163021] [ 8] [b20n05:121824] [ 0] [0x1000000504d8] +[b20n05:121824] [ 1] [0x3fffec506500] +[b20n05:121824] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n05:121824] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n05:121824] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n13:168159] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n13:168159] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n13:168159] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n13:168159] [10] [0x1000000504d8] +[a30n11:163257] [ 1] [0x3fffc0ba7eb0] +[a30n11:163257] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n11:163257] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n11:163257] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n11:163257] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n15:151249] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n12:178716] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n12:178716] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n10:134731] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n10:134731] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n17:168163] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n17:168163] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n17:168163] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n17:168163] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n17:168163] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n04:168487] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n13:166922] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n13:166922] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n13:166922] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n13:166922] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n14:150131] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n14:150131] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n17:125807] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n17:125807] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n15:120060] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n15:120060] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n15:120060] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n15:120060] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n15:120060] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n15:120060] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n03:158658] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n03:158658] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n03:158658] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n03:158658] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n03:158658] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n11:153418] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n11:153418] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n11:153418] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n11:153418] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n11:153418] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n11:153418] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n11:153418] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n11:153418] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n11:153418] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n11:153418] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n11:153418] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n11:153418] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n16:128442] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n16:128442] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n16:128442] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n16:128442] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n16:128442] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n13:123883] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n15:124799] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n15:124799] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n15:124799] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n15:124799] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n15:124799] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n15:124799] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n15:124799] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n15:124799] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b22n02:119530] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b22n02:119530] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n16:173547] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n16:173547] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n05:164941] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n05:164941] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n05:164941] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n05:164941] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n05:164941] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n05:164941] *** End of error message *** +[a27n08:54624] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a27n08:54624] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n08:54624] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n18:164800] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n18:08430] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n18:08430] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n12:132102] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a33n02:167596] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n02:167596] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n18:131582] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n18:131582] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n18:131582] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n18:131582] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n18:131582] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n18:131582] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n18:131582] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n18:131582] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n18:131582] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n18:131582] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n18:131582] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n12:120127] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n08:131376] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n06:167371] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n14:133063] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n03:123618] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n03:123618] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n03:123618] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n03:123618] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n03:123618] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n03:123618] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n03:123618] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n03:123618] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n03:123618] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n16:165924] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n16:165924] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n16:165924] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a29n16:157886] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a29n16:157886] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n16:157886] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n16:157886] [10] [0x1000000504d8] +[b21n07:117362] [ 1] [0x3fffc71799d0] +[b21n07:117362] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b21n07:117362] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n07:117362] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n07:117362] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n07:117362] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n07:117362] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n07:117362] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n07:117362] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n07:133375] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n07:133375] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n07:133375] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n07:133375] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n07:133375] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n07:133375] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n07:133375] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n07:133375] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n07:133375] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n11:160350] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n11:160350] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n09:123415] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n09:123415] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n09:123415] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n09:123415] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n09:123415] *** End of error message *** +[b20n01:128827] Signal: Segmentation fault (11) +[b20n01:128827] Signal code: (3) +[b20n01:128827] Failing at address: 0xb170000000000a0 +[b20n01:128827] [ 0] [0x1000000504d8] +[b20n01:128827] [ 1] [0x3ffff87a2a10] +[b20n01:128827] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n01:128827] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n02:06560] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n02:06560] [ 6] [b23n07:118164] [ 8] [0x3fffe192a5c0] +[b07n14:176066] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n14:176066] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n14:176066] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n14:176066] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n16:127349] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n16:127349] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n16:127349] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n12:12762] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n12:12762] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n12:12762] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n12:12762] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n12:12762] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n12:122967] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n12:122967] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n12:122967] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n12:122967] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n12:122967] *** End of error message *** +[0x1000000504d8] +[b02n17:03842] [ 1] [0x3fffc6d85d30] +[b02n17:03842] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n17:03842] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n17:03842] [ 4] [b02n07:180190] *** Process received signal *** +[b02n07:180190] Signal: Segmentation fault (11) +[b02n07:180190] Signal code: (3) +[b02n07:180190] Failing at address: 0xb170000000000a0 +[b02n07:180190] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n08:08953] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n08:08953] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n08:08953] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n17:149906] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n17:149906] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n06:126376] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n06:126376] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n18:126907] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n18:126907] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n18:126907] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n18:126907] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n18:126907] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n05:135163] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n01:141815] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n01:141815] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n06:143008] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n07:132087] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n07:132087] *** End of error message *** +[0x1000000504d8] +[b03n06:01836] [ 1] [0x3fffd48e4390] +[b03n06:01836] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n06:01836] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n06:01836] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n01:167300] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n01:167300] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n17:150930] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n17:150930] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n17:150930] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n17:150930] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n17:150930] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n17:150930] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n17:150930] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n04:169909] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n04:169909] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n04:169909] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n04:169909] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n10:176307] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n10:176307] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n02:164491] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n02:164491] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n02:164491] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n02:164491] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n07:138204] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n01:128155] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n01:128155] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n01:128155] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n11:137870] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n11:137870] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n11:137870] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n11:137870] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n16:01570] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n16:01570] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n14:161619] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n14:161619] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a28n11:01173] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n11:01173] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n13:123706] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n13:123706] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n13:123706] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n13:123706] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n11:152984] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n16:122491] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n16:122491] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n16:122491] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n16:122491] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n16:122491] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n12:156915] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n11:171961] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n11:171961] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n11:171961] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n11:171961] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n11:171961] *** End of error message *** +[a30n12:148377] *** Process received signal *** +[a30n12:148377] Signal: Segmentation fault (11) +[a30n12:148377] Signal code: (3) +[a30n12:148377] Failing at address: 0xb170000000000a0 +[a30n12:148377] [ 0] [0x1000000504d8] +[a30n12:148377] [ 1] [0x3fffc808f710] +[a30n12:148377] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n12:148377] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n12:11142] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n12:11142] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n12:11142] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n12:11142] [13] [a30n16:153289] *** Process received signal *** +[a30n16:153289] Signal: Segmentation fault (11) +[a30n16:153289] Signal code: (3) +[a30n16:153289] Failing at address: 0xb170000000000a0 +[a30n16:153289] [ 0] [0x1000000504d8] +[a30n16:153289] [ 1] [0x3ffffde22280] +[a30n16:153289] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n16:153289] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n16:153289] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n04:175996] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n04:175996] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b12n01:145731] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b12n01:145731] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b12n01:145731] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b12n01:145731] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b12n01:145731] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n15:148583] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n15:148583] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n02:163318] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n02:163318] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n02:163318] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n02:163318] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n09:123810] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n09:123810] [ 9] [b10n17:164563] *** End of error message *** +[0x1000000504d8] +[b14n07:127358] [ 1] [0x3fffe0234340] +[b14n07:127358] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b14n07:127358] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n07:127358] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n07:127358] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n07:127358] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n07:127358] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n07:127358] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n07:127358] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n07:127358] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n07:127358] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n07:127358] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n07:127358] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n07:127358] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n07:127358] *** End of error message *** +[b19n08:121689] *** Process received signal *** +[b19n08:121689] Signal: Segmentation fault (11) +[b19n08:121689] Signal code: (3) +[b19n08:121689] Failing at address: 0x14170000000000a0 +[b19n08:121689] [ 0] [0x1000000504d8] +[b19n08:121689] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n17:146797] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n17:146797] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n17:146797] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n15:137005] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n13:163195] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n13:163195] [13] [b07n13:165259] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n08:163097] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n08:163097] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n08:163097] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n08:163097] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n08:163097] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n03:168933] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n03:168933] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n06:177007] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n06:177007] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n06:177007] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n06:177007] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n11:120239] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n11:120239] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n11:120239] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n02:149230] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n02:149230] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n02:149230] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n02:149230] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n01:138007] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n01:138007] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n01:138007] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n01:138007] [10] [a30n14:163547] *** Process received signal *** +[a30n14:163547] Signal: Segmentation fault (11) +[a30n14:163547] Signal code: (3) +[a30n14:163547] Failing at address: 0xb170000000000a0 +[a30n14:163547] [ 0] [0x1000000504d8] +[a30n14:163547] [ 1] [0x3fffc5edd770] +[a30n14:163547] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n14:163547] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n14:163547] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n14:163547] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n14:163547] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n14:163547] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n14:163547] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n14:163547] [ 9] [b03n11:175593] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n03:135162] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n06:10223] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n06:10223] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n03:136664] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n03:136664] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n03:136664] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n03:136664] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n03:136664] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n02:178476] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n02:178476] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n02:178476] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n02:178476] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n03:125243] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n01:65724] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n01:65724] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n01:65724] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n01:65724] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n06:154620] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n06:154620] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n06:154620] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n06:154620] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n05:162568] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n05:162568] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n05:162568] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n05:162568] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n04:169702] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n04:169702] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n04:169702] [13] [b06n15:161754] *** Process received signal *** +[b06n15:161754] Signal: Segmentation fault (11) +[b06n15:161754] Signal code: (3) +[b06n15:161754] Failing at address: 0x5170000000000a0 +[b06n15:161754] [ 0] [0x1000000504d8] +[b06n15:161754] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n03:01355] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n03:130348] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n11:148671] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n11:148671] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n11:148671] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n11:148671] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n17:119465] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n17:119465] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n17:119465] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n17:119465] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n17:119465] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n05:135120] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n05:135120] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n05:135120] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n05:135120] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n05:135120] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n03:151882] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n03:151882] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n02:123703] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n02:123703] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n02:123703] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n01:164961] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n01:164961] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n01:164961] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n01:164961] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n17:118572] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n17:118572] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n17:118572] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n17:118572] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n17:163816] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n17:163816] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n17:163816] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n17:163816] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n17:163816] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n01:141343] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n01:141343] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n02:147882] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n02:147882] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n02:147882] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n02:147882] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n02:147882] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n05:131262] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n01:179259] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n01:179259] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n01:179259] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n01:179259] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n01:179259] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n01:179259] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n02:122558] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n02:122558] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n02:122558] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n02:122558] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n03:153853] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n03:153853] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n03:153853] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n03:153853] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n03:153853] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n03:153853] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n03:153853] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n04:04846] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n04:04846] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n04:04846] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n06:171500] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n01:165123] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n01:165123] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n01:128001] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n01:128001] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n01:128001] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n01:128001] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n05:120330] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n05:120330] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n16:164552] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n16:164552] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n16:164552] [ 8] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b15n06:120730] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n06:120730] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n06:120730] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n06:120730] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n06:120730] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n06:120730] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n11:160795] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n10:16970] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n10:16970] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n10:16970] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n10:16970] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n10:16970] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n10:16970] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n10:16970] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n11:03982] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n11:03982] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n11:03982] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n12:178602] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n18:179167] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n18:179167] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n18:179167] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n18:179167] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b23n06:117529] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n13:123513] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b13n11:134521] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n11:134521] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n04:158634] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a33n17:168152] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n17:168152] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n17:168152] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n17:168152] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n17:168152] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n17:168152] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b08n14:178322] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n03:154292] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n03:154292] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n03:154292] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n03:154292] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n17:176411] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n17:176411] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n17:176411] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n01:163390] *** End of error message *** +[b08n07:168960] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n10:118389] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n10:118389] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n10:118389] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a29n05:174779] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n05:174779] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n05:174779] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n05:174779] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n05:174779] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n05:174779] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n05:174779] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n01:96752] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n01:96752] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n01:96752] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n03:06822] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n10:169206] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a36n05:139909] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n07:165087] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n07:165087] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n07:165087] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n07:165087] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n09:170654] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n09:170654] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n09:170654] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n08:146118] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n08:146118] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n08:146118] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n08:146118] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n08:146118] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n14:152745] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n14:152745] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n14:152745] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n14:152745] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n14:152745] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n14:152745] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n15:120129] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n01:170763] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n01:170763] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n01:170763] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n01:170763] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n01:170763] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n16:124957] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n16:124957] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n02:122177] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n02:122177] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n02:122177] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n02:122177] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n16:154182] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n16:154182] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n16:154182] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n16:154182] [10] [b07n05:176907] *** Process received signal *** +[b07n05:176907] Signal: Segmentation fault (11) +[b07n05:176907] Signal code: (3) +[b07n05:176907] Failing at address: 0xb170000000000a0 +[b07n05:176907] [ 0] [0x1000000504d8] +[b07n05:176907] [ 1] [0x3fffebe384f0] +[b07n05:176907] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n05:176907] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n05:176907] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n05:176907] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n05:176907] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n05:176907] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n05:176907] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n05:176907] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n05:176907] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n05:176907] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n05:176907] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n05:176907] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n05:176907] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n05:176907] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n18:128542] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n08:127442] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n08:127442] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n02:125371] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n02:125371] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n18:10724] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n11:156151] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n11:156151] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n11:156151] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n11:156151] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n11:156151] *** End of error message *** +[0x3fffccd3b840] +[b07n12:160960] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n12:160960] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n12:160960] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n07:156490] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n07:156490] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n10:120974] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n10:120974] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n10:120974] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n10:122760] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n10:122760] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n10:122760] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n12:122873] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n12:122873] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n12:122873] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n12:122873] *** End of error message *** +[0x1000000504d8] +[b04n07:02837] [ 1] [0x3ffff9e40df0] +[b04n07:02837] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b04n07:02837] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b04n07:02837] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n03:135034] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n03:135034] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n03:135034] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n03:135034] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n03:135034] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n14:126122] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n14:126122] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n14:126122] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n14:126122] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n14:126122] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n14:126122] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n14:126122] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n14:126122] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n14:126122] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n01:123572] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n01:123572] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n01:123572] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n01:123572] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n01:123572] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n01:123572] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n01:123572] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n01:123572] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n01:123572] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n01:123572] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n01:123572] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n04:177595] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n04:177595] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n04:177595] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n04:177595] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n04:177595] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n08:165284] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n08:165284] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n08:165284] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n07:125448] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n07:125448] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n07:125448] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n07:125448] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n08:124117] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n08:124117] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n08:124117] [ 8] [b20n03:124679] *** Process received signal *** +[b20n03:124679] Signal: Segmentation fault (11) +[b20n03:124679] Signal code: (3) +[b20n03:124679] Failing at address: 0xb170000000000a0 +[b20n03:124679] [ 0] [0x1000000504d8] +[b20n03:124679] [ 1] [0x3ffff3f0e6f0] +[b20n03:124679] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n03:124679] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n03:124679] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n03:124679] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n03:124679] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n03:124679] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n03:124679] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n03:124679] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n03:124679] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n07:163250] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n07:163250] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n07:163250] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n11:128957] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n11:128957] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n11:128957] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n11:128957] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n11:128957] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n15:148164] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n15:148164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n15:148164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n15:148164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n15:148164] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n08:144851] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n08:144851] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n08:144851] [ 9] [b19n18:122551] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n18:122551] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n09:143896] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n15:32399] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n15:32399] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n12:164379] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n12:164379] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n12:164379] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n12:164379] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n01:163022] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n16:130373] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n16:130373] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n16:130373] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n16:130373] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n13:168159] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n13:168159] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n13:168159] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n13:168159] [14] [b20n05:121825] [ 0] [0x1000000504d8] +[b20n05:121825] [ 1] [0x3fffe0f97eb0] +[b20n05:121825] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n05:121825] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n05:121825] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n15:151249] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n11:163258] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n11:163258] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n10:134730] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n10:134730] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n10:134730] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n10:134730] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n12:178717] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n14:150130] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n13:166923] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n13:166923] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n13:166923] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n13:166923] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n13:166923] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n13:166923] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n13:166923] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n13:166923] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n17:125806] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n17:125806] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n17:125806] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n17:125806] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n17:125806] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n17:125806] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n17:125806] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n03:158659] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n03:158659] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n03:158659] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n03:158659] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b13n16:128441] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n16:128441] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n16:128441] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n16:128441] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b22n02:119530] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b22n02:119530] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b22n02:119530] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n02:119530] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n02:119530] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n02:119530] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n02:119530] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n02:119530] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n02:119530] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n15:124798] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n15:124798] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n15:124798] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n15:124798] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n15:124798] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n15:124798] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n15:124798] [13] [b19n13:123887] *** Process received signal *** +[b19n13:123887] Signal: Segmentation fault (11) +[b19n13:123887] Signal code: (3) +[b19n13:123887] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a27n08:54623] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n08:54623] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n08:54623] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n16:173546] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n16:173546] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n12:132102] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n12:132102] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n02:167595] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n02:167595] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n02:167595] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n02:167595] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n02:167595] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b14n18:131581] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b14n18:131581] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n08:131377] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n08:131377] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n03:123617] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n03:123617] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n03:123617] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n14:133063] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n14:133063] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a29n16:157887] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n16:157887] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n16:157887] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n16:157887] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n16:157887] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b21n07:117361] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b21n07:117361] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b21n07:117361] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b21n07:117361] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b21n07:117361] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b21n07:117361] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n07:117361] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n07:117361] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n07:117361] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n07:133376] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b05n08:08952] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b05n08:08952] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b05n08:08952] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n18:126906] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n18:126906] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n18:126906] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n18:126906] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n18:126906] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n16:165923] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n16:165923] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n16:165923] [ 7] [0x1000000504d8] +[b20n01:128828] [ 1] [0x3fffc2645df0] +[b20n01:128828] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n01:128828] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n01:128828] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n01:128828] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n01:128828] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n07:118164] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n07:118164] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n07:118164] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n07:118164] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n07:118164] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n07:118164] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n07:118164] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n14:176067] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n14:176067] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n14:176067] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n16:127350] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n12:12763] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n12:12763] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n12:12763] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b22n12:122966] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b22n12:122966] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b22n12:122966] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n12:122966] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n12:122966] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n17:03842] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n17:03842] [ 6] [0x1000000504d8] +[b02n07:180190] [ 1] [0x3fffec146d10] +[b02n07:180190] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n07:180190] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n07:180190] [ 4] [a34n17:149902] *** Process received signal *** +[a34n17:149902] Signal: Segmentation fault (11) +[a34n17:149902] Signal code: (3) +[a34n17:149902] Failing at address: 0x14170000000000a0 +[a34n17:149902] [ 0] [0x1000000504d8] +[a34n17:149902] [ 1] [0x3fffd9da5b60] +[a34n17:149902] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n17:149902] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n17:149902] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n17:149902] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n17:149902] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n17:149902] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n17:149902] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n17:149902] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n17:149902] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n17:149902] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n17:149902] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n17:149902] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n17:149902] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n17:149902] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n02:06559] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n02:06559] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n02:06559] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n02:06559] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n09:123416] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n09:123416] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n09:123416] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n09:123416] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n09:123416] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n01:141814] *** End of error message *** +[ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n06:126376] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n06:126376] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n06:126376] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n06:01837] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n01:167301] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n01:167301] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n01:167301] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n01:167301] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n01:167301] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n04:169910] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n04:169910] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n02:164492] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n10:176306] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n07:138205] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n07:138205] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n01:128155] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n01:128155] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n11:137869] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n11:137869] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b11n14:161620] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n16:01568] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n16:01568] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n16:01568] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n16:01568] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n16:01568] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n11:01174] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n11:01174] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n11:01174] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n11:01174] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n13:123705] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n13:123705] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n11:152984] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n11:152984] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n16:122492] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n12:148377] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n16:153288] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n16:153288] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n16:153288] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n16:153288] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n12:11143] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n12:11143] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n12:11143] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n12:11143] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n12:11143] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n01:145730] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n01:145730] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n02:163319] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n02:163319] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n02:163319] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n02:163319] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n09:123810] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n08:121685] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n08:121685] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n17:146796] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n17:146796] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n17:146796] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n17:146796] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n17:146796] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n15:137004] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n13:165258] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n13:165258] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n13:165258] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n13:163195] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n06:177006] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n06:177006] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n06:177006] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n06:177006] [10] [ 1] [0x3fffc3df1ea0] +[a35n02:149231] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a35n02:149231] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a35n02:149231] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a35n02:149231] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n14:163546] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n14:163546] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n14:163546] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n14:163546] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n01:138006] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n01:138006] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n01:138006] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n01:138006] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n01:138006] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n03:135162] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n06:10222] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n06:10222] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n06:10222] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n06:10222] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n02:178477] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n02:178477] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n02:178477] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n03:136665] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a26n01:65725] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a26n01:65725] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a26n01:65725] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a26n01:65725] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n01:65725] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n01:65725] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n06:154619] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n06:154619] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n06:154619] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n06:154619] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n06:154619] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n06:154619] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n06:154619] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n05:162569] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n05:162569] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n05:162569] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n05:162569] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n04:169703] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n15:161753] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n15:161753] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n02:122559] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n02:122559] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n02:122559] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n02:122559] [14] [b01n03:01357] *** Process received signal *** +[b01n03:01357] Signal: Segmentation fault (11) +[b01n03:01357] Signal code: (3) +[b01n03:01357] Failing at address: 0xb170000000000a0 +[b01n03:01357] [ 0] [0x1000000504d8] +[b01n03:01357] [ 1] [0x3ffff27edb80] +[b01n03:01357] [ 2] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n11:148672] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n11:148672] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n11:148672] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n11:148672] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n11:148672] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a32n03:151883] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n03:151883] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n03:151883] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n03:151883] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n03:151883] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n03:151883] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n03:151883] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n03:151883] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n03:151883] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n02:123703] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n02:123703] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n01:164960] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n01:164960] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n01:164960] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n01:164960] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n01:164960] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n03:130348] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n03:130348] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n03:130348] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n17:118571] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n17:118571] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n17:118571] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n17:118571] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n01:179260] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n01:179260] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n01:179260] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n01:179260] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n01:141344] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n02:147882] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n05:131262] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n05:131262] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n05:131262] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n04:04847] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n04:04847] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n01:128002] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n05:120329] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b10n16:164553] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b10n16:164553] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b10n16:164553] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n16:164553] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n16:164553] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b15n06:120731] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b15n06:120731] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b15n06:120731] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n06:120731] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n06:120731] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n06:120731] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n06:120731] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n06:120731] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n06:120731] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n06:120731] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n11:03982] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n11:03982] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a28n18:179166] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n18:179166] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b23n06:117529] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n12:178602] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n13:123513] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n11:134520] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n11:134520] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n11:134520] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a31n04:158635] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a31n04:158635] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n04:158635] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n17:168153] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n17:168153] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n17:168153] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n17:168153] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n17:168153] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n17:168153] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n17:168153] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n03:154292] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n03:154292] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n03:154292] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n14:178323] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n14:178323] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n14:178323] [ 8] [a30n01:163387] *** Process received signal *** +[a30n01:163387] Signal: Segmentation fault (11) +[a30n01:163387] Signal code: (3) +[a30n01:163387] Failing at address: 0xb170000000000a0 +[a30n01:163387] [ 0] [0x1000000504d8] +[a30n01:163387] [ 1] [0x3ffff0564060] +[a30n01:163387] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n01:163387] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n17:176411] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n17:176411] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b22n10:118389] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n10:169207] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n10:169207] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n01:96752] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n01:96752] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n03:06821] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n03:06821] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n03:06821] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n03:06821] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n03:06821] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n03:06821] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n03:06821] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n03:06821] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a36n05:139909] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a36n05:139909] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n05:139909] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n07:165088] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n07:165088] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n12:160959] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n12:160959] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n11:156152] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n11:156152] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n11:156152] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n11:156152] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a32n14:152744] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a32n14:152744] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a32n14:152744] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a32n14:152744] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n14:152744] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n14:152744] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n14:152744] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n14:152744] [14] [0x3fffd62a1c10] +[b03n01:170764] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b03n01:170764] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b03n01:170764] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n01:170764] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n01:170764] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n01:170764] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n01:170764] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n01:170764] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n16:124957] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n16:124957] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n16:124957] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n02:122176] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n02:122176] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n02:122176] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n02:122176] [14] [0x3fffdf7d5300] +[b07n05:176908] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n05:176908] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n05:176908] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n05:176908] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n05:176908] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n05:176908] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n05:176908] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n05:176908] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n05:176908] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n05:176908] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n05:176908] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n05:176908] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n05:176908] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n05:176908] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n16:154182] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n16:154182] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n16:154182] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n16:154182] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n02:125372] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n02:125372] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n02:125372] [ 6] [a02n08:146114] *** Process received signal *** +[a02n08:146114] Signal: Segmentation fault (11) +[a02n08:146114] Signal code: (3) +[a02n08:146114] Failing at address: 0x14170000000000a0 +[a02n08:146114] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n09:170655] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n09:170655] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n09:170655] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n09:170655] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n08:127443] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n07:156490] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n18:10723] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n18:10723] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n18:128542] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n18:128542] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n18:128542] [13] [b17n03:135034] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n10:122759] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n10:122759] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n10:122759] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n10:120975] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n10:120975] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n10:120975] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n12:122874] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b04n07:02837] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n14:126121] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n14:126121] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n14:126121] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n14:126121] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n04:177594] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n04:177594] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n04:177594] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n04:177594] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n01:123573] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n01:123573] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n01:123573] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n01:123573] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n01:123573] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n01:123573] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n01:123573] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n01:123573] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n01:123573] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n07:125447] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n07:125447] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n07:125447] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n07:125447] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n07:125447] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n07:125447] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n08:124117] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n08:124117] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n03:124678] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n03:124678] *** End of error message *** +[b18n11:128956] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n11:128956] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n11:128956] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n11:128956] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n15:148164] *** End of error message *** +/lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b07n07:163251] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n07:163251] [ 4] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n08:144852] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n08:144852] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n08:144852] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n08:144852] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n08:144852] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n18:122552] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n18:122552] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n18:122552] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n01:163021] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n01:163021] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n01:163021] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n12:164379] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n12:164379] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n12:164379] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n16:130374] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n16:130374] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n16:130374] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n13:168159] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n05:121824] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n05:121824] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n05:121824] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n05:121824] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n11:163257] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n11:163257] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n11:163257] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n11:163257] [ 9] [b09n12:178716] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a36n15:151249] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b14n18:131581] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b14n18:131581] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b14n18:131581] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b14n18:131581] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n13:166922] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n17:125807] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n17:125807] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n17:125807] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n17:125807] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n17:125807] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n03:158658] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n03:158658] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n03:158658] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n16:128442] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n16:128442] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n15:124799] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n15:124799] *** End of error message *** +[b19n13:123887] [ 0] [0x1000000504d8] +[b19n13:123887] [ 1] [0x3fffe852d2b0] +[b19n13:123887] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n13:123887] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a27n08:54624] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a27n08:54624] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n08:54624] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n08:54624] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n16:173547] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n16:173547] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n12:132102] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n12:132102] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n08:131376] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n08:131376] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n08:131376] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n08:131376] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a29n16:157886] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a29n16:157886] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a29n16:157886] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a29n16:157886] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a33n02:167596] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a33n02:167596] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a33n02:167596] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a33n02:167596] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n10:134731] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n10:134731] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n10:134731] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n09:123416] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n02:06560] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n02:06560] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n02:06560] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n02:06560] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n14:133063] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n14:133063] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n14:133063] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n16:165924] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n16:165924] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n16:165924] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b21n07:117362] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b21n07:117362] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b21n07:117362] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n07:117362] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n07:117362] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n07:117362] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n07:133375] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n01:128827] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n01:128827] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n16:127349] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n16:127349] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n16:127349] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n16:127349] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n16:127349] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n14:176066] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n14:176066] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n14:176066] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n14:176066] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n12:12762] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n12:12762] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n12:12762] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n12:12762] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n12:122966] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n08:08953] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n08:08953] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n08:08953] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n08:08953] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n17:03842] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n17:03842] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n17:03842] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n18:126907] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n16:122491] *** End of error message *** +[b02n07:180189] *** Process received signal *** +[b02n07:180189] Signal: Segmentation fault (11) +[b02n07:180189] Signal code: (3) +[b02n07:180189] Failing at address: 0xb170000000000a0 +[b02n07:180189] [ 0] [0x1000000504d8] +[b02n07:180189] [ 1] [0x3fffe9177330] +[b02n07:180189] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b02n07:180189] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n01:141815] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n01:141815] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n01:141815] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n01:141815] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b03n06:01836] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n06:01836] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n06:01836] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n06:01836] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n06:126376] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n06:126376] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n06:126376] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n06:126376] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n06:126376] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n06:126376] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n06:126376] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n01:167300] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n01:167300] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n01:167300] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n01:167300] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n01:167300] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n04:169909] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n04:169909] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n04:169909] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n04:169909] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n11:137870] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n01:128156] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n01:128156] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n01:128156] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a28n11:01173] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n11:01173] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n11:01173] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n11:01173] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n11:01173] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n13:123706] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b11n14:161620] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n16:01569] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n16:01569] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n11:152984] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n11:152984] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n11:152984] [10] [a34n17:149903] *** Process received signal *** +[a34n17:149903] Signal: Segmentation fault (11) +[a34n17:149903] Signal code: (3) +[a34n17:149903] Failing at address: 0x14170000000000a0 +[a34n17:149903] [ 0] [0x1000000504d8] +[a34n17:149903] [ 1] [0x3fffc8a6b3a0] +[a34n17:149903] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a34n17:149903] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a34n17:149903] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a34n17:149903] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a34n17:149903] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a34n17:149903] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a34n17:149903] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a34n17:149903] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a34n17:149903] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a34n17:149903] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a34n17:149903] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a34n17:149903] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n17:149903] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n17:149903] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n12:148377] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n16:153289] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n16:153289] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n12:11142] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n12:11142] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b12n01:145731] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b12n01:145731] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n02:163319] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n02:163319] *** End of error message *** +[0x3fffd959d800] +[b19n08:121689] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n08:121689] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n08:121689] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n09:123810] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n17:146797] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n17:146797] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n17:146797] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n17:146797] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n13:165258] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n13:165258] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n13:165258] [14] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n13:163195] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n06:177007] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n02:149230] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n02:149230] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n14:163547] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n14:163547] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n14:163547] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n14:163547] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n14:163547] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n14:163547] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n01:138007] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n01:138007] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n01:138007] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n01:138007] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n01:138007] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n03:135162] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n06:10223] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n06:10223] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n06:10223] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n06:10223] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n03:136664] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n02:178476] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n02:178476] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n02:178476] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n02:178476] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n02:178476] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a26n01:65724] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a26n01:65724] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n05:162568] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n05:162568] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n05:162568] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n05:162568] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n05:162568] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n04:169702] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n04:169702] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n02:122558] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n02:122558] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n02:122558] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n02:122558] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n02:122558] [14] [0x3fffef672330] +[b06n15:161754] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b06n15:161754] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b06n15:161754] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b10n16:164552] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b10n16:164552] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n16:164552] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n11:148671] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n11:148671] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n11:148671] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a32n03:151882] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a32n03:151882] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a32n03:151882] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n03:151882] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n03:151882] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n02:123703] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n02:123703] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n02:123703] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n02:123703] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n02:123703] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n01:164961] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n01:164961] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n03:130348] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n17:118572] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n17:118572] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n01:179259] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n01:179259] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n01:179259] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n05:131262] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n05:131262] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n05:131262] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n05:131262] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n05:131262] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n05:131262] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n01:128001] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n01:128001] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n01:128001] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n01:128001] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n01:128001] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n01:128001] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n06:120731] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n06:120731] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b15n05:120330] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b15n05:120330] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n05:120330] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n04:04846] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n04:04846] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n04:04846] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n03:01355] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n03:01355] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n03:01355] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n11:03983] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n11:03983] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n11:03983] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n11:03983] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n11:03983] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n11:03983] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n11:03983] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n18:179167] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n18:179167] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n18:179167] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n18:179167] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n18:179167] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n12:178602] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b23n06:117529] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n13:123513] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n13:123513] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n13:123513] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n13:123513] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n13:123513] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n13:123513] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n13:123513] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n13:123513] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n13:123513] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b13n11:134521] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b13n11:134521] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b13n11:134521] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n11:134521] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n11:134521] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a31n04:158634] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n04:158634] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n04:158634] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n17:168152] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n17:168152] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n17:168152] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n17:168152] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n17:168152] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n03:154292] *** End of error message *** +[a30n01:163388] *** Process received signal *** +[a30n01:163388] Signal: Segmentation fault (11) +[a30n01:163388] Signal code: (3) +[a30n01:163388] Failing at address: 0xb170000000000a0 +[a30n01:163388] [ 0] [0x1000000504d8] +[a30n01:163388] [ 1] [0x3fffcaa44d40] +[a30n01:163388] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a30n01:163388] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n01:96753] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n01:96753] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n01:96753] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n01:96753] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n01:96753] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b22n10:118389] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n10:169206] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n14:178323] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n14:178323] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n17:176411] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n17:176411] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n17:176411] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n14:152747] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n14:152747] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n07:165087] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n12:160960] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n12:160960] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n12:160960] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n12:160960] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n12:160960] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n11:156152] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n11:156152] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n05:139909] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n05:139909] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n05:139909] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n05:139909] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n05:139909] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n05:139909] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n05:139909] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n16:124958] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n16:124958] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n16:124958] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n16:124958] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n16:124958] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n16:124958] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n16:124958] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n16:124958] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n16:124958] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n16:124958] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n02:122177] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n01:170763] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n01:170763] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n01:170763] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n01:170763] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n01:170763] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n01:170763] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n16:154181] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n16:154181] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n16:154181] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n16:154181] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n02:125371] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n02:125371] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n02:125371] [ 9] [0x1000000504d8] +[a02n08:146114] [ 1] [0x3fffee0a0a10] +[a02n08:146114] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n08:146114] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n08:146114] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n09:170654] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n07:156489] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n07:156489] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n07:156489] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n10:122760] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n10:122760] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n10:122760] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n10:122760] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n10:122760] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n18:128542] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n18:128542] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n10:120974] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n10:120974] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n10:120974] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n10:120974] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n10:120974] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n10:120974] *** End of error message *** +[b19n12:122871] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b04n07:02837] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b04n07:02837] [ 7] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n14:126122] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n04:177594] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n04:177594] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n07:125448] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n07:125448] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n07:125448] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n07:125448] *** End of error message *** +[b17n08:124118] *** Process received signal *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n03:124679] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n03:124679] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n03:124679] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n03:124679] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n03:124679] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n09:143896] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n11:128957] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n07:163250] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n07:163250] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n18:122551] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n18:122551] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a32n12:164379] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n12:164379] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n08:144851] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n08:144851] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n08:144851] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n08:144851] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n08:144851] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n16:130373] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n16:130373] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n01:163021] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n01:163021] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n01:163021] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n11:163258] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n11:163258] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n11:163258] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n11:163258] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n05:121825] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n05:121825] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n05:121825] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a36n15:151249] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a36n15:151249] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a36n15:151249] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a36n15:151249] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a36n15:151249] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a36n15:151249] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a36n15:151249] *** End of error message *** +[b09n12:178715] *** Process received signal *** +[b09n12:178715] Signal: Segmentation fault (11) +[b09n12:178715] Signal code: (3) +[b09n12:178715] Failing at address: 0x14170000000000a0 +[b09n12:178715] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n13:166923] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n13:166923] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n10:134730] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n10:134730] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n10:134730] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n16:128441] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n16:128441] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n16:128441] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n17:125807] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n17:125807] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n17:125807] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n17:125807] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n17:125807] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n13:123887] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n12:12763] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n12:12763] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n12:12763] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n12:12763] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a27n08:54623] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a27n08:54623] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n16:173546] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n16:173546] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n16:173546] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n16:173546] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n16:173546] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n12:132102] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n12:132102] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n12:132102] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n12:132102] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n12:132102] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n02:167595] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n02:167595] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n02:167595] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n08:131377] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n08:131377] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n08:131377] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n08:131377] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n16:157887] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b14n18:131581] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n18:131581] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n18:131581] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n18:131581] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n18:131581] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n02:06559] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n02:06559] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n02:06559] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n02:06559] [13] [b19n14:133061] *** Process received signal *** +[b19n14:133061] Signal: Segmentation fault (11) +[b19n14:133061] Signal code: (3) +[b19n14:133061] Failing at address: 0xb170000000000a0 +[b19n14:133061] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n16:165923] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n16:165923] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n16:165923] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n16:165923] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b21n07:117361] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b21n07:117361] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b21n07:117361] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n01:128828] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n01:128828] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n01:128828] [ 9] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n15:124798] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n15:124798] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n14:176067] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n14:176067] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n14:176067] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b05n08:08952] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b05n08:08952] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b05n08:08952] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b05n08:08952] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n01:141815] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n17:03842] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n07:180190] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n06:01837] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n06:01837] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n06:01837] [ 8] [b19n06:126375] *** Process received signal *** +[b19n06:126375] Signal: Segmentation fault (11) +[b19n06:126375] Signal code: (3) +[b19n06:126375] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n04:169910] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n04:169910] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n04:169910] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n04:169910] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n04:169910] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n01:167301] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n01:128155] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n01:128155] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n01:128155] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n01:128155] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n11:01174] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n11:01174] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n11:01174] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b11n14:161620] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b11n14:161620] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n11:152984] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n11:152984] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n11:152984] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n12:148377] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n16:153289] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n16:153289] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n16:153289] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n12:11143] *** End of error message *** +[b19n08:121688] *** Process received signal *** +[b19n08:121688] Signal: Segmentation fault (11) +[b19n08:121688] Signal code: (3) +[b19n08:121688] Failing at address: 0x14170000000000a0 +[b19n08:121688] [ 0] [0x1000000504d8] +[b19n08:121688] [ 1] [0x3fffc5f284a0] +[b19n08:121688] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n08:121688] [ 3] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n17:146796] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n17:146796] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n17:146796] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n17:146796] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n09:123810] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n09:123810] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b07n13:165259] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n06:177006] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n06:177006] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n06:177006] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n06:177006] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n06:177006] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a35n02:149231] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a35n02:149231] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a35n02:149231] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a35n02:149231] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n02:149231] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n14:163546] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n14:163546] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n14:163546] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n14:163546] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a35n03:135162] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n06:10222] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n06:10222] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n06:10222] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n06:10222] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n02:178477] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n02:178477] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n02:178477] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n02:178477] [14] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n05:162569] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n05:162569] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n04:169703] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b01n03:01357] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n03:01357] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n03:01357] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n02:122559] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n03:151883] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b06n15:161754] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n11:148672] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n11:148672] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n11:148672] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n11:148672] *** End of error message *** +[b19n02:123705] *** Process received signal *** +[b19n02:123705] Signal: Segmentation fault (11) +[b19n02:123705] Signal code: (3) +[b19n02:123705] Failing at address: 0x14170000000000a0 +[b19n02:123705] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a34n01:164960] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a34n01:164960] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b14n03:130348] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n11:03982] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n11:03982] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n11:03982] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n11:03982] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n11:03982] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b03n04:04847] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b03n04:04847] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b03n04:04847] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n01:179260] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n01:179260] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n05:120330] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n05:120330] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n05:120330] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n05:120330] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b15n06:120730] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b15n06:120730] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b15n06:120730] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b15n06:120730] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b15n06:120730] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n06:120730] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n06:120730] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b10n16:164553] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n16:164553] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n16:164553] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n16:164553] [14] [b19n05:131259] *** Process received signal *** +[b19n05:131259] Signal: Segmentation fault (11) +[b19n05:131259] Signal code: (3) +[b19n05:131259] Failing at address: 0xb170000000000a0 +[b19n05:131259] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a28n18:179166] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a28n18:179166] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a28n18:179166] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n12:178602] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b23n06:117529] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b23n06:117529] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b23n06:117529] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b23n06:117529] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b23n06:117529] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b23n06:117529] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b23n06:117529] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b23n06:117529] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b23n06:117529] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n02:125372] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n02:125372] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n02:125372] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n02:125372] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n04:158634] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n04:158634] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n01:163387] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n01:163387] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n01:163387] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n01:163387] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n01:163387] [ 8] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b22n10:118389] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n01:96752] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n01:96752] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n01:96752] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b08n14:178322] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b08n14:178322] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b08n14:178322] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b08n14:178322] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b08n14:178322] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n14:178322] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n14:178322] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n14:178322] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n14:178322] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n10:169207] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n10:169207] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n10:169207] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n10:169207] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n10:169207] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n10:169207] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n10:169207] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a32n14:152744] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n17:176411] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n02:122176] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n12:160959] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n12:160959] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n12:160959] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n12:160959] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n16:124957] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n16:124957] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n01:170764] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n01:170764] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n01:170764] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n01:170764] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n01:170764] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n01:170764] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n16:154181] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n11:134520] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n11:134520] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n11:134520] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n11:134520] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n11:134520] [13] [a02n08:146115] *** Process received signal *** +[a02n08:146115] Signal: Segmentation fault (11) +[a02n08:146115] Signal code: (3) +[a02n08:146115] Failing at address: 0xb170000000000a0 +[a02n08:146115] [ 0] [0x1000000504d8] +[a02n08:146115] [ 1] [0x3fffe0bbdbb0] +[a02n08:146115] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[a02n08:146115] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a02n08:146115] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n07:156490] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n09:170655] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n09:170655] [11] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n10:122759] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n10:122759] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n18:10723] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n10:120975] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n10:120975] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n10:120975] *** End of error message *** +[b19n12:122870] *** Process received signal *** +[b19n12:122870] Signal: Segmentation fault (11) +[b19n12:122870] Signal code: (3) +[b19n12:122870] Failing at address: 0xb170000000000a0 +[b19n12:122870] [ 0] [0x1000000504d8] +[b19n12:122870] [ 1] [0x3fffed65baf0] +[b19n12:122870] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n12:122870] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b04n07:02837] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b04n07:02837] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b04n07:02837] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n07:125447] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n07:125447] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n07:125447] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n08:124117] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n08:124117] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n08:124117] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n08:124117] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n08:124117] *** End of error message *** +[b13n09:143896] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n11:128956] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n11:128956] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n11:128956] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n11:128956] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n11:128956] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n07:163251] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n07:163251] [ 6] [b19n18:122555] *** Process received signal *** +[b19n18:122555] Signal: Segmentation fault (11) +[b19n18:122555] Signal code: (3) +[b19n18:122555] Failing at address: 0x14170000000000a0 +[b19n18:122555] [ 0] [0x1000000504d8] +[b19n18:122555] [ 1] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n08:144852] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n08:144852] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n08:144852] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n08:144852] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n16:130374] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n16:130374] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n16:130374] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n16:130374] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n16:130374] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n01:163022] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n01:163022] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n01:163022] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n11:163257] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n11:163257] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n11:163257] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n11:163257] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n05:121824] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n05:121824] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n05:121824] [11] [0x1000000504d8] +[b09n12:178715] [ 1] [0x3ffffe0f5540] +[b09n12:178715] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n12:178715] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n12:178715] [ 4] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n16:128441] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n16:128441] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n16:128441] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n16:128441] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b13n10:134731] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b13n10:134731] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n10:134731] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n10:134731] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n10:134731] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n10:134731] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n17:125806] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n17:125806] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n17:125806] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n17:125806] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n17:125806] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n13:123887] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n13:123887] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a29n16:157886] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n02:167595] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n02:167595] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n16:173546] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n16:173546] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n16:173546] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n16:173546] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n08:131376] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n08:131376] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n08:131376] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n08:131376] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n18:131581] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n02:06560] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n02:06560] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n02:06560] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n02:06560] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n16:165924] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n16:165924] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n16:165924] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n16:165924] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n16:165924] [13] [b19n14:133060] *** Process received signal *** +[b19n14:133060] Signal: Segmentation fault (11) +[b19n14:133060] Signal code: (3) +[b19n14:133060] Failing at address: 0xb170000000000a0 +[b19n14:133060] [ 0] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n08:08953] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n08:08953] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n01:128827] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n01:128827] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n01:128827] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n12:12763] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n12:12763] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n14:176066] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n14:176066] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n14:176066] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b02n07:180189] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b02n07:180189] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n17:03842] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n17:03842] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n17:03842] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n17:03842] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n06:01836] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n06:01836] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n06:01836] [11] [b19n06:126375] [ 0] [0x1000000504d8] +[b19n06:126375] [ 1] [0x3fffc71f57f0] +[b19n06:126375] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n06:126375] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n01:128156] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n01:128156] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n01:128156] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n01:128156] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n04:169909] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n04:169909] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b11n14:161620] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n12:148377] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n12:148377] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n11:152984] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n11:152984] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n11:01173] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n11:01173] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n11:01173] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n11:01173] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n08:121689] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n08:121689] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n16:153289] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n17:146797] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n17:146797] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n17:146797] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n13:165258] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n09:123810] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n02:149231] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n02:149231] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n02:149231] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n02:149231] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n02:149231] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n14:163546] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n14:163546] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n14:163546] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a35n03:135162] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n06:10223] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n06:10223] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n06:10223] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n18:179167] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n02:122558] *** End of error message *** +[ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b06n15:161754] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b06n15:161754] [ 7] [b19n02:123706] *** Process received signal *** +[b19n02:123706] Signal: Segmentation fault (11) +[b19n02:123706] Signal code: (3) +[b19n02:123706] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n03:01355] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n03:01355] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n03:01355] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b14n03:130348] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b14n03:130348] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b14n03:130348] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b14n03:130348] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n11:03983] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n11:03983] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n04:04846] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n04:04846] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n04:04846] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n04:04846] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b15n05:120330] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b15n05:120330] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b10n16:164552] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b10n16:164552] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b10n16:164552] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n16:164552] *** End of error message *** +[b19n05:131260] *** Process received signal *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n02:178476] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b13n11:134521] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n11:134521] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n11:134521] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n11:134521] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n12:178602] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a31n04:158635] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a31n04:158635] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a31n04:158635] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a31n04:158635] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n01:163387] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n01:163387] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n01:163387] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n01:163387] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n01:163387] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n01:96752] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n10:169206] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n10:169206] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n10:169206] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b08n14:178323] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b08n14:178323] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b08n14:178323] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b08n14:178323] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n14:178323] *** End of error message *** +[b20n02:122180] *** Process received signal *** +[b20n02:122180] Signal: Segmentation fault (11) +[b20n02:122180] Signal code: (3) +[b20n02:122180] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n12:160960] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n12:160960] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n12:160960] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n12:160960] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n02:125371] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n02:125371] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n02:125371] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n02:125371] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n02:125371] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n16:154182] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n08:146115] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n08:146115] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n09:170654] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n09:170654] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n09:170654] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n09:170654] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n12:122870] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n07:156489] [14] [a30n18:10724] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n18:10724] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b04n07:02837] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b04n07:02837] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b04n07:02837] [13] [b17n08:124118] Signal: Segmentation fault (11) +[b17n08:124118] Signal code: (3) +[b17n08:124118] Failing at address: 0x5170000000000a0 +[b17n08:124118] [ 0] [0x1000000504d8] +[b17n08:124118] [ 1] [0x3fffc173db10] +[b17n08:124118] [0x3ffff5106b00] +[b19n18:122555] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n18:122555] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n18:122555] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n08:144851] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n11:163258] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n11:163258] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n11:163258] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n11:163258] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n11:163258] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n01:163021] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n07:163250] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n07:163250] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n07:163250] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n07:163250] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n16:130373] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n16:130373] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n16:130373] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n16:130373] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n16:130373] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n12:178715] [ 5] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b13n10:134730] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b13n10:134730] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n10:134730] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n10:134730] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n05:121824] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n05:121824] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n05:121824] [14] [b19n13:123886] *** Process received signal *** +[b19n13:123886] Signal: Segmentation fault (11) +[b19n13:123886] Signal code: (3) +[b19n13:123886] Failing at address: 0x14170000000000a0 +[b19n13:123886] [ 0] [0x1000000504d8] +[b19n13:123886] [ 1] [0x3ffffd5cca00] +[b19n13:123886] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n13:123886] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n08:131377] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n08:131377] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n08:131377] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n08:131377] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a33n02:167596] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a33n02:167596] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a33n02:167596] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a33n02:167596] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a33n02:167596] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a33n02:167596] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b05n08:08952] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b05n08:08952] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b05n08:08952] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b05n08:08952] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n02:06559] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n02:06559] *** End of error message *** +[0x1000000504d8] +[b19n14:133061] [ 1] [0x3fffe57efc10] +[b19n14:133061] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n14:133061] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n14:133061] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n14:133061] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n14:133061] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n14:133061] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n14:133061] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n14:133061] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n14:133061] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n14:133061] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n14:133061] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n14:133061] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n14:133061] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n14:133061] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n16:165923] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n16:165923] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n16:165923] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n16:165923] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n01:128828] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n01:128828] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n01:128828] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n01:128828] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n14:176067] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n14:176067] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n14:176067] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n14:176067] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n17:03842] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n06:01837] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n06:01837] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n06:01837] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n06:01837] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n06:01837] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n07:180190] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n07:180190] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n07:180190] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n07:180190] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n06:126375] [ 4] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n01:128155] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b11n14:161620] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n12:148377] [ 9] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n11:01174] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n11:01174] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n11:01174] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n08:121685] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n08:121685] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n08:121685] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n08:121685] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n08:121685] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n08:121685] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n16:153289] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n16:153289] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n16:153289] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n16:153289] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b07n13:165259] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n09:123810] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a35n03:135162] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a35n03:135162] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a35n03:135162] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a35n03:135162] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n06:10222] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n02:178477] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n04:169703] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b01n03:01356] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b01n03:01356] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n03:01356] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n03:01356] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n03:01356] [ 8] [0x1000000504d8] +[b19n02:123705] [ 1] [0x3fffc1be6e30] +[b19n02:123705] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n02:123705] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n02:123705] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b06n15:161754] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b06n15:161754] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b06n15:161754] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b06n15:161754] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b06n15:161754] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b06n15:161754] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n15:161754] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n15:161754] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b03n04:04847] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b03n04:04847] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b03n04:04847] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n04:04847] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n04:04847] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n04:04847] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a28n18:179166] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a28n18:179166] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a28n18:179166] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a28n18:179166] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a28n18:179166] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n18:179166] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b13n11:134520] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n11:134520] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n12:178602] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n12:178602] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n12:178602] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n12:178602] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n12:178602] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n12:178602] *** End of error message *** +[0x1000000504d8] +[b19n05:131259] [ 1] [0x3ffff1cfd6a0] +[b19n05:131259] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n05:131259] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n05:131259] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[a30n01:163388] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a30n01:163388] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a30n01:163388] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a30n01:163388] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a30n01:163388] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a30n01:163388] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n01:163388] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n01:163388] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n01:163388] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n01:163388] [13] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b10n16:164553] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n04:158634] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n04:158634] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n04:158634] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n04:158634] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n04:158634] *** End of error message *** +[b04n01:96752] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n10:169207] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n10:169207] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b08n14:178322] *** End of error message *** +[b20n02:122179] *** Process received signal *** +[b20n02:122179] Signal: Segmentation fault (11) +[b20n02:122179] Signal code: (3) +[b20n02:122179] Failing at address: 0x14170000000000a0 +[b20n02:122179] [ 0] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n02:125372] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n02:125372] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n02:125372] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n02:125372] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[a02n08:146114] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[a02n08:146114] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n12:160959] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n12:160959] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n12:160959] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n12:160959] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n09:170655] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n09:170655] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n09:170655] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n09:170655] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n07:156490] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n07:156490] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n07:156490] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n07:156490] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n07:156490] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n18:10723] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n18:10723] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n18:10723] *** End of error message *** +[ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b17n08:124118] [ 3] [b19n12:122871] Signal: Segmentation fault (11) +[b19n12:122871] Signal code: (3) +[b19n12:122871] Failing at address: 0xb170000000000a0 +[b19n12:122871] [ 0] [0x1000000504d8] +[b19n12:122871] [ 1] [0x3fffc3dfde50] +[b19n12:122871] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n12:122871] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b04n07:02837] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b04n07:02837] *** End of error message *** +[b19n18:122554] *** Process received signal *** +[b19n18:122554] Signal: Segmentation fault (11) +[b19n18:122554] Signal code: (3) +[b19n18:122554] Failing at address: 0x14170000000000a0 +[b19n18:122554] [ 0] [0x1000000504d8] +[b19n18:122554] [ 1] [0x3fffd8c21cc0] +[b19n18:122554] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n18:122554] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n18:122554] [ 4] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n11:163257] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n11:163257] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n08:144852] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n01:163022] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n01:163022] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n01:163022] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n01:163022] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n01:163022] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n01:163022] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n07:163251] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n07:163251] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n07:163251] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n07:163251] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n16:130374] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n16:130374] *** End of error message *** +[b09n12:178714] *** Process received signal *** +[b09n12:178714] Signal: Segmentation fault (11) +[b09n12:178714] Signal code: (3) +[b09n12:178714] Failing at address: 0xb170000000000a0 +[b09n12:178714] [ 0] [0x1000000504d8] +[b09n12:178714] [ 1] [0x3fffff906620] +[b09n12:178714] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b09n12:178714] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n10:134731] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n13:123886] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n13:123886] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n08:131376] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n05:121825] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n05:121825] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n05:121825] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n05:121825] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n05:121825] [12] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n02:06560] *** End of error message *** +[0x1000000504d8] +[b19n14:133060] [ 1] [0x3fffddaaa100] +[b19n14:133060] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n14:133060] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n14:133060] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n14:133060] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n14:133060] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n14:133060] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n14:133060] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n14:133060] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n14:133060] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n14:133060] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n14:133060] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n14:133060] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n14:133060] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n14:133060] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n01:128827] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n01:128827] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n01:128827] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n01:128827] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n01:128827] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b06n16:165924] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b06n16:165924] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n14:176066] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n14:176066] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n14:176066] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n06:01837] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n06:01837] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n01:128156] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n01:128156] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n01:128156] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n01:128156] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n01:128156] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b02n07:180189] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b02n07:180189] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b02n07:180189] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b02n07:180189] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n06:126375] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b11n14:161620] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b11n14:161620] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b11n14:161620] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b11n14:161620] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b11n14:161620] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n12:148377] [10] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a28n11:01173] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n08:121688] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n08:121688] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n08:121688] [ 6] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n16:153289] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b07n13:165259] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n03:01355] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n03:01355] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n03:01355] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n03:01355] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n03:01355] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n04:169703] [ 7] [b19n02:123706] [ 0] [0x1000000504d8] +[b19n02:123706] [ 1] [0x3fffdcddb610] +[b19n02:123706] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n02:123706] [ 3] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n04:04847] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b13n11:134521] *** End of error message *** +[b19n05:131260] Signal: Segmentation fault (11) +[b19n05:131260] Signal code: (3) +[b19n05:131260] Failing at address: 0xb170000000000a0 +[b19n05:131260] [ 0] [0x1000000504d8] +[b19n05:131260] [ 1] [0x3fffd8a01dc0] +[b19n05:131260] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n05:131260] [ 3] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n01:163387] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n01:163387] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a31n04:158635] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a31n04:158635] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a31n04:158635] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a31n04:158635] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n04:158635] *** End of error message *** +[b20n02:122180] [ 0] [0x1000000504d8] +[b20n02:122180] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a30n10:169206] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n10:169206] [11] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n02:125371] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n08:146115] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n08:146115] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n08:146115] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n08:146115] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n12:160960] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n12:160960] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n09:170654] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a31n07:156489] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b17n08:124118] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n12:122870] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n12:122870] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n12:122870] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n12:122870] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n18:10724] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n18:10724] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n18:122555] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n18:122555] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n12:178715] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n12:178715] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n12:178715] [ 8] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n01:163022] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n01:163022] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n07:163250] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n13:123887] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n13:123887] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n13:123887] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n08:131377] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n05:121824] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n01:128828] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n01:128828] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b03n06:01836] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b03n06:01836] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b03n06:01836] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b03n06:01836] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n07:180190] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n07:180190] [11] [b18n01:128152] *** Process received signal *** +[b18n01:128152] Signal: Segmentation fault (11) +[b18n01:128152] Signal code: (3) +[b18n01:128152] Failing at address: 0x14170000000000a0 +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n06:126375] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n06:126375] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n06:126375] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a30n12:148377] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n08:121689] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n08:121689] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n08:121689] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n08:121689] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n08:121689] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n08:121689] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n08:121689] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n08:121689] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n04:169703] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b01n03:01357] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b01n03:01357] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b01n03:01357] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n03:01357] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n03:01357] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n03:01357] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b07n13:165259] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b07n13:165259] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n02:123705] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n02:123705] [ 6] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n01:163388] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n01:163388] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n13:123886] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n13:123886] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n13:123886] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n13:123886] [ 9] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n02:125372] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n10:169206] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n10:169206] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n10:169206] [14] [0x1000000504d8] +[b20n02:122179] [ 1] [0x3fffefc006a0] +[b20n02:122179] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n02:122179] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n02:122179] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n02:122179] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[a02n08:146114] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[a02n08:146114] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[a02n08:146114] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[a02n08:146114] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n08:146114] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n08:146114] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b17n08:124118] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n12:160959] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n12:122871] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n12:122871] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n18:122554] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n18:122554] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n18:122554] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n18:122554] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n18:122554] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n07:163251] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n07:163251] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n07:163251] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n07:163251] [14] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n05:121825] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n05:121825] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n05:121825] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n01:128827] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n01:128827] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b09n12:178714] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b09n12:178714] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n05:131259] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n05:131259] [ 6] [b18n01:128153] *** Process received signal *** +[b18n01:128153] Signal: Segmentation fault (11) +[b18n01:128153] Signal code: (3) +[b18n01:128153] Failing at address: 0x14170000000000a0 +[b18n01:128153] [ 0] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a30n12:148377] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a30n12:148377] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a30n12:148377] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n12:148377] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n04:169703] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b02n07:180189] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b02n07:180189] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n08:121685] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n08:121685] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n08:121685] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n06:126375] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n06:126375] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b07n13:165259] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b07n13:165259] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b07n13:165259] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b07n13:165259] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b07n13:165259] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b07n13:165259] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n13:165259] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b01n03:01356] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n03:01356] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n03:01356] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n03:01356] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n03:01356] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n03:01356] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n03:01356] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n05:131259] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n05:131259] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n05:131259] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n05:131259] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n05:131259] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n05:131259] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n05:131259] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n05:131259] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n05:131259] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n02:123706] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n02:123706] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[a02n08:146115] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[a02n08:146115] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n08:146115] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n08:146115] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a30n10:169206] *** End of error message *** +[ 1] [0x3ffff546dfd0] +[b20n02:122180] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b20n02:122180] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b20n02:122180] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b17n08:124118] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n18:122555] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n18:122555] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n18:122555] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n18:122555] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n12:122870] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n12:122870] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n12:122870] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n12:122870] [12] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n08:121688] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n08:121688] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n08:121688] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n08:121688] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n08:121688] [11] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n12:178715] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n12:178715] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n12:178715] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n12:178715] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n12:178715] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n12:178715] [14] [b18n01:128152] [ 0] [0x1000000504d8] +[b18n01:128152] [ 1] [0x3fffc22ebb70] +[b18n01:128152] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n01:128152] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n01:128152] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n13:123887] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n13:123887] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n13:123887] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n13:123887] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n13:123887] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n13:123887] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b01n04:169703] [10] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n07:180190] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n07:180190] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n07:180190] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n07:180190] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b07n07:163251] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n06:126375] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n06:126375] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n06:126375] [13] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[a02n08:146114] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[a02n08:146114] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n08:146114] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n03:01357] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n03:01357] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n03:01357] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n03:01357] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n02:123706] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n02:123706] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n02:123706] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n05:131260] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n05:131260] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n05:131260] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n05:131260] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n05:131260] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n05:131260] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n05:131260] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n05:131260] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n05:131260] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n05:131260] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n05:131260] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n05:131260] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n02:122179] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n02:122179] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n02:122179] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n02:122179] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b17n08:124118] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n12:122871] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n12:122871] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n12:122871] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n12:122871] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n18:122554] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n18:122554] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n18:122554] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n18:122554] [13] [0x1000000504d8] +[b18n01:128153] [ 1] [0x3ffffb3a8550] +[b18n01:128153] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b18n01:128153] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b18n01:128153] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n01:128153] [ 5] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n08:121689] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b09n12:178714] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b09n12:178714] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b09n12:178714] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b09n12:178714] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b09n12:178714] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b09n12:178714] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b09n12:178714] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b09n12:178714] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b09n12:178714] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n12:178714] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n13:123886] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n13:123886] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n13:123886] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n13:123886] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n13:123886] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b01n04:169703] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b02n07:180189] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b02n07:180189] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b02n07:180189] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b02n07:180189] *** End of error message *** +/lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n06:126375] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n06:126375] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n18:122555] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n18:122555] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n18:122555] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n18:122555] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n18:122555] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n02:123705] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n02:123705] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n02:123705] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n02:123705] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n18:122554] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n18:122554] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b17n08:124118] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b17n08:124118] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b17n08:124118] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b17n08:124118] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b17n08:124118] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b17n08:124118] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b17n08:124118] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b17n08:124118] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b20n02:122180] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b20n02:122180] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b20n02:122180] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b20n02:122180] [ 8] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n12:122870] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n12:122870] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n12:122870] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b18n01:128152] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n01:128152] [ 6] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n08:121688] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n08:121688] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n08:121688] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n08:121688] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b09n12:178715] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n13:123886] *** End of error message *** +/ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b01n04:169703] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b01n04:169703] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n02:122179] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n02:122179] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n02:122179] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n02:122179] [13] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n02:123706] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n02:123706] [10] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n02:122179] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n02:122179] *** End of error message *** +[0x1000000504d8] +[b19n06:126377] [ 1] [0x3fffd2726ea0] +[b19n06:126377] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n06:126377] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n06:126377] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n12:122871] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n12:122871] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n12:122871] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n12:122871] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n12:122871] [14] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b18n01:128153] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n01:128153] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n01:128153] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b20n02:122180] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b20n02:122180] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b20n02:122180] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b20n02:122180] [12] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b01n04:169703] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b01n04:169703] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n02:123705] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n02:123705] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n02:123705] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n02:123705] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n02:123705] *** End of error message *** +[b19n06:126378] *** Process received signal *** +[b19n06:126378] Signal: Segmentation fault (11) +[b19n06:126378] Signal code: (3) +[b19n06:126378] Failing at address: 0x14170000000000a0 +[b19n06:126378] [ 0] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n12:122871] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b18n01:128152] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b18n01:128152] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n01:128152] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n01:128152] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n01:128152] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n01:128152] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n01:128152] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n01:128152] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n01:128152] *** End of error message *** +/lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[a02n08:146115] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n02:123706] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n02:123706] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n02:123706] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n02:123706] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n02:123706] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b18n01:128153] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b18n01:128153] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b18n01:128153] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b20n02:122180] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b20n02:122180] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b20n02:122180] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n06:126377] [ 5] [0x1000000504d8] +[b19n06:126378] [ 1] [0x3fffe66d0130] +[b19n06:126378] [ 2] /lib64/libmlx5-rdmav2.so(+0x3b89c)[0x10000223b89c] +[b19n06:126378] [ 3] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(+0xb6f7c)[0x1000031d6f7c] +[b19n06:126378] [ 4] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV13GpuPutMessage11advance_gpuEv+0x2dc)[0x1000031f38cc] +[b19n06:126378] [ 5] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n06:126378] [ 6] [b19n06:126377] *** Process received signal *** +[b19n06:126377] Signal: Segmentation fault (11) +[b19n06:126377] Signal code: (3) +[b19n06:126377] Failing at address: 0x14170000000000a0 +[b19n06:126377] [ 0] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b18n01:128153] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b18n01:128153] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b18n01:128153] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b18n01:128153] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(_ZN4PAMI6Device3IBV16GpuProgressQueue7Message9__advanceEPvS4_+0x2c)[0x1000031dddac] +[b19n06:126377] [ 6] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n06:126377] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n06:126377] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n06:126377] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n06:126377] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n06:126377] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n06:126377] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n06:126377] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n06:126377] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n06:126377] *** End of error message *** +/autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/pami_port/libpami.so.3(PAMI_Context_advancev+0x128)[0x1000031d8d98] +[b19n06:126378] [ 7] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/spectrum_mpi/mca_pml_pami.so(mca_pml_pami_progress+0x34)[0x1000030e6804] +[b19n06:126378] [ 8] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libopen-pal.so.3(opal_progress+0x70)[0x100000b7fc00] +[b19n06:126378] [ 9] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(ompi_request_default_wait_all+0x124)[0x100000352b14] +[b19n06:126378] [10] /autofs/nccs-svm1_sw/summit/.swci/1-compute/opt/spack/20171006/linux-rhel7-ppc64le/gcc-6.4.0/spectrum-mpi-10.2.0.0-20180110-ly2nyvslpq5p6o5ndkhrcagfhvdg3a3q/lib/libmpi_ibm.so.3(PMPI_Waitall+0xf0)[0x10000039bee0] +[b19n06:126378] [11] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(_ZN19ScaLBL_Communicator11RecvD3Q19AAEPd+0x34)[0x1002ce84] +[b19n06:126378] [12] /ccs/home/mcclurej/summit/build/LBPM-WIA/tests/TestCommD3Q19(main+0xc88)[0x10010168] +[b19n06:126378] [13] /lib64/libc.so.6(+0x24980)[0x100000794980] +[b19n06:126378] [14] /lib64/libc.so.6(__libc_start_main+0xc4)[0x100000794b74] +[b19n06:126378] *** End of error message *** +User defined signal 2 +ERROR: The connection to the JSM daemon has failed. + This is a strong indication that the JSM daemon has exited unexpectedly. + When this occurs, error logs are left on the affected nodes in +/tmp/JSMDIAG-LOG-- or in +/tmp/jsm.batch4.9413/52581/JSMD-LOG-- +[batch4:107195] PMIX ERROR: UNPACK-PAST-END in file tool/pmix_tool.c at line 105 ------------------------------------------------------------ -Sender: LSF System -Subject: Job 43296: in cluster Done +Sender: LSF System +Subject: Job 43348: in cluster Exited -Job was submitted from host by user in cluster at Fri Mar 16 11:28:16 2018 -Job was executed on host(s) <1*batch3>, in queue , as user in cluster at Fri Mar 16 11:29:16 2018 - <42*a07n04> - <42*a07n05> - <42*a07n10> - <42*a07n11> - <42*a07n12> - <42*a07n13> - <42*a07n14> - <42*a07n15> - <42*a07n16> - <42*a07n17> - <42*a08n01> - <42*a08n02> - <42*a08n03> - <42*a08n04> - <42*a08n05> - <42*a08n06> - <42*a08n07> - <42*a08n08> - <42*a08n09> - <42*a08n10> - <42*a08n11> - <42*a08n12> - <42*a08n14> - <42*a08n15> - <42*a08n16> - <42*a08n17> - <42*a08n18> - <42*a09n02> - <42*a09n03> - <42*a09n04> - <42*a09n05> - <42*a09n06> - <42*a09n07> - <42*a09n08> - <42*a09n09> - <42*a09n10> - <42*a09n11> - <42*a09n13> - <42*a09n14> - <42*a09n15> - <42*a09n16> - <42*a09n17> - <42*a09n18> - <42*a10n01> - <42*a10n02> - <42*a10n03> - <42*a10n04> - <42*a10n05> - <42*a10n06> - <42*a10n07> - <42*a10n08> - <42*a10n09> - <42*a10n10> - <42*a10n11> - <42*a10n12> - <42*a10n13> - <42*a10n14> - <42*a10n15> - <42*a10n16> - <42*a10n17> - <42*a10n18> - <42*a11n01> - <42*a11n02> - <42*a11n03> - <42*a11n04> - <42*a11n05> - <42*a15n02> - <42*a15n03> - <42*a15n04> - <42*a15n05> - <42*a15n06> - <42*a15n07> - <42*a15n08> - <42*a15n09> - <42*a15n10> - <42*a15n11> - <42*a15n12> - <42*a15n13> - <42*a15n14> - <42*a15n15> - <42*a15n16> - <42*a15n17> - <42*a15n18> - <42*a16n01> - <42*a16n02> - <42*a16n03> - <42*a16n04> - <42*a16n05> - <42*a16n06> - <42*a16n07> - <42*a16n08> - <42*a16n09> - <42*a16n10> - <42*a16n11> - <42*a16n12> - <42*a16n13> - <42*a16n14> - <42*a16n15> - <42*a16n16> - <42*a16n17> +Job was submitted from host by user in cluster at Fri Mar 16 12:44:03 2018 +Job was executed on host(s) <1*batch4>, in queue , as user in cluster at Fri Mar 16 12:45:46 2018 + <42*a02n03> + <42*a02n04> + <42*a02n06> + <42*a02n07> + <42*a02n08> + <42*a02n09> + <42*a02n10> + <42*a02n18> + <42*a26n01> + <42*a26n02> + <42*a26n03> + <42*a26n04> + <42*a26n05> + <42*a26n06> + <42*a26n07> + <42*a26n08> + <42*a26n09> + <42*a26n10> + <42*a26n11> + <42*a26n12> + <42*a26n13> + <42*a26n14> + <42*a26n15> + <42*a26n16> + <42*a26n17> + <42*a26n18> + <42*a27n01> + <42*a27n02> + <42*a27n03> + <42*a27n04> + <42*a27n05> + <42*a27n06> + <42*a27n07> + <42*a27n08> + <42*a27n09> + <42*a27n10> + <42*a27n11> + <42*a27n13> + <42*a27n14> + <42*a27n15> + <42*a27n17> + <42*a27n18> + <42*a28n01> + <42*a28n02> + <42*a28n03> + <42*a28n04> + <42*a28n05> + <42*a28n06> + <42*a28n07> + <42*a28n08> + <42*a28n09> + <42*a28n10> + <42*a28n11> + <42*a28n12> + <42*a28n13> + <42*a28n14> + <42*a28n15> + <42*a28n16> + <42*a28n17> + <42*a28n18> + <42*a29n01> + <42*a29n02> + <42*a29n03> + <42*a29n04> + <42*a29n05> + <42*a29n06> + <42*a29n07> + <42*a29n08> + <42*a29n09> + <42*a29n10> + <42*a29n11> + <42*a29n12> + <42*a29n13> + <42*a29n15> + <42*a29n16> + <42*a29n17> + <42*a29n18> + <42*a30n01> + <42*a30n02> + <42*a30n03> + <42*a30n04> + <42*a30n05> + <42*a30n06> + <42*a30n07> + <42*a30n08> + <42*a30n09> + <42*a30n10> + <42*a30n11> + <42*a30n12> + <42*a30n13> + <42*a30n14> + <42*a30n15> + <42*a30n16> + <42*a30n17> + <42*a30n18> + <42*a31n01> + <42*a31n02> + <42*a31n03> + <42*a31n04> + <42*a31n05> + <42*a31n06> + <42*a31n07> + <42*a31n08> + <42*a31n09> + <42*a31n10> + <42*a31n11> + <42*a31n12> + <42*a31n13> + <42*a31n14> + <42*a31n15> + <42*a31n16> + <42*a31n17> + <42*a31n18> + <42*a32n01> + <42*a32n02> + <42*a32n03> + <42*a32n04> + <42*a32n05> + <42*a32n06> + <42*a32n07> + <42*a32n08> + <42*a32n09> + <42*a32n10> + <42*a32n11> + <42*a32n12> + <42*a32n13> + <42*a32n14> + <42*a32n15> + <42*a32n16> + <42*a32n17> + <42*a32n18> + <42*a33n01> + <42*a33n02> + <42*a33n03> + <42*a33n04> + <42*a33n05> + <42*a33n06> + <42*a33n07> + <42*a33n08> + <42*a33n09> + <42*a33n10> + <42*a33n11> + <42*a33n12> + <42*a33n13> + <42*a33n14> + <42*a33n15> + <42*a33n16> + <42*a33n17> + <42*a33n18> + <42*a34n01> + <42*a34n02> + <42*a34n03> + <42*a34n05> + <42*a34n06> + <42*a34n07> + <42*a34n08> + <42*a34n09> + <42*a34n10> + <42*a34n11> + <42*a34n12> + <42*a34n13> + <42*a34n14> + <42*a34n16> + <42*a34n17> + <42*a34n18> + <42*a35n01> + <42*a35n02> + <42*a35n03> + <42*a35n04> + <42*a35n05> + <42*a35n06> + <42*a35n07> + <42*a35n08> + <42*a35n09> + <42*a35n10> + <42*a35n11> + <42*a35n12> + <42*a35n13> + <42*a35n14> + <42*a35n15> + <42*a35n16> + <42*a35n17> + <42*a35n18> + <42*a36n01> + <42*a36n02> + <42*a36n03> + <42*a36n04> + <42*a36n05> + <42*a36n06> + <42*a36n07> + <42*a36n09> + <42*a36n10> + <42*a36n11> + <42*a36n12> + <42*a36n13> + <42*a36n14> + <42*a36n15> + <42*a36n17> + <42*a36n18> + <42*b01n01> + <42*b01n02> + <42*b01n03> + <42*b01n04> + <42*b01n05> + <42*b01n06> + <42*b01n07> + <42*b01n08> + <42*b01n09> + <42*b01n10> + <42*b01n11> + <42*b01n12> + <42*b01n13> + <42*b01n14> + <42*b01n15> + <42*b01n16> + <42*b01n17> + <42*b01n18> + <42*b02n01> + <42*b02n02> + <42*b02n03> + <42*b02n04> + <42*b02n05> + <42*b02n06> + <42*b02n07> + <42*b02n09> + <42*b02n10> + <42*b02n11> + <42*b02n12> + <42*b02n13> + <42*b02n14> + <42*b02n15> + <42*b02n16> + <42*b02n17> + <42*b02n18> + <42*b03n01> + <42*b03n02> + <42*b03n03> + <42*b03n04> + <42*b03n05> + <42*b03n06> + <42*b03n07> + <42*b03n09> + <42*b03n10> + <42*b03n11> + <42*b03n12> + <42*b03n13> + <42*b03n14> + <42*b03n15> + <42*b03n16> + <42*b03n17> + <42*b03n18> + <42*b04n01> + <42*b04n02> + <42*b04n03> + <42*b04n04> + <42*b04n05> + <42*b04n06> + <42*b04n07> + <42*b04n08> + <42*b04n09> + <42*b04n10> + <42*b04n11> + <42*b04n12> + <42*b04n13> + <42*b04n14> + <42*b04n15> + <42*b04n16> + <42*b04n17> + <42*b04n18> + <42*b05n01> + <42*b05n02> + <42*b05n03> + <42*b05n04> + <42*b05n05> + <42*b05n06> + <42*b05n07> + <42*b05n08> + <42*b05n09> + <42*b05n10> + <42*b05n11> + <42*b05n12> + <42*b05n13> + <42*b05n14> + <42*b05n15> + <42*b05n16> + <42*b05n17> + <42*b05n18> + <42*b06n01> + <42*b06n02> + <42*b06n03> + <42*b06n04> + <42*b06n05> + <42*b06n06> + <42*b06n07> + <42*b06n08> + <42*b06n09> + <42*b06n10> + <42*b06n11> + <42*b06n12> + <42*b06n13> + <42*b06n14> + <42*b06n15> + <42*b06n16> + <42*b06n17> + <42*b06n18> + <42*b07n01> + <42*b07n02> + <42*b07n03> + <42*b07n04> + <42*b07n05> + <42*b07n06> + <42*b07n07> + <42*b07n08> + <42*b07n09> + <42*b07n10> + <42*b07n11> + <42*b07n12> + <42*b07n13> + <42*b07n14> + <42*b07n15> + <42*b07n16> + <42*b07n17> + <42*b08n01> + <42*b08n02> + <42*b08n03> + <42*b08n04> + <42*b08n05> + <42*b08n06> + <42*b08n07> + <42*b08n08> + <42*b08n09> + <42*b08n10> + <42*b08n11> + <42*b08n12> + <42*b08n13> + <42*b08n14> + <42*b08n15> + <42*b08n16> + <42*b08n17> + <42*b08n18> + <42*b09n01> + <42*b09n02> + <42*b09n03> + <42*b09n04> + <42*b09n05> + <42*b09n06> + <42*b09n07> + <42*b09n08> + <42*b09n09> + <42*b09n10> + <42*b09n11> + <42*b09n12> + <42*b09n13> + <42*b09n14> + <42*b09n15> + <42*b09n16> + <42*b09n17> + <42*b09n18> + <42*b10n01> + <42*b10n02> + <42*b10n03> + <42*b10n04> + <42*b10n05> + <42*b10n06> + <42*b10n07> + <42*b10n08> + <42*b10n10> + <42*b10n11> + <42*b10n12> + <42*b10n13> + <42*b10n14> + <42*b10n15> + <42*b10n16> + <42*b10n17> + <42*b10n18> + <42*b11n03> + <42*b11n04> + <42*b11n05> + <42*b11n06> + <42*b11n07> + <42*b11n09> + <42*b11n10> + <42*b11n11> + <42*b11n12> + <42*b11n13> + <42*b11n14> + <42*b11n15> + <42*b11n16> + <42*b11n17> + <42*b11n18> + <42*b12n01> + <42*b12n02> + <42*b12n03> + <42*b12n04> + <42*b12n05> + <42*b12n06> + <42*b12n07> + <42*b12n08> + <42*b12n09> + <42*b12n10> + <42*b12n11> + <42*b12n12> + <42*b12n13> + <42*b12n14> + <42*b12n15> + <42*b12n16> + <42*b12n17> + <42*b12n18> + <42*b13n01> + <42*b13n02> + <42*b13n03> + <42*b13n04> + <42*b13n05> + <42*b13n06> + <42*b13n07> + <42*b13n08> + <42*b13n09> + <42*b13n10> + <42*b13n11> + <42*b13n12> + <42*b13n13> + <42*b13n14> + <42*b13n15> + <42*b13n16> + <42*b13n17> + <42*b13n18> + <42*b14n01> + <42*b14n02> + <42*b14n03> + <42*b14n04> + <42*b14n05> + <42*b14n06> + <42*b14n07> + <42*b14n08> + <42*b14n09> + <42*b14n10> + <42*b14n11> + <42*b14n12> + <42*b14n13> + <42*b14n14> + <42*b14n15> + <42*b14n16> + <42*b14n17> + <42*b14n18> + <42*b15n01> + <42*b15n02> + <42*b15n03> + <42*b15n04> + <42*b15n05> + <42*b15n06> + <42*b15n07> + <42*b15n08> + <42*b15n09> + <42*b15n10> + <42*b15n11> + <42*b15n12> + <42*b15n13> + <42*b15n14> + <42*b15n15> + <42*b15n16> + <42*b15n17> + <42*b15n18> + <42*b16n01> + <42*b16n02> + <42*b16n03> + <42*b16n04> + <42*b16n05> + <42*b16n06> + <42*b16n07> + <42*b16n08> + <42*b16n09> + <42*b16n10> + <42*b16n11> + <42*b16n12> + <42*b16n13> + <42*b16n14> + <42*b16n16> + <42*b16n17> + <42*b16n18> + <42*b17n01> + <42*b17n02> + <42*b17n03> + <42*b17n04> + <42*b17n05> + <42*b17n06> + <42*b17n07> + <42*b17n08> + <42*b17n09> + <42*b17n10> + <42*b17n11> + <42*b17n12> + <42*b17n13> + <42*b17n14> + <42*b17n15> + <42*b17n16> + <42*b17n17> + <42*b17n18> + <42*b18n01> + <42*b18n02> + <42*b18n03> + <42*b18n04> + <42*b18n05> + <42*b18n06> + <42*b18n07> + <42*b18n08> + <42*b18n09> + <42*b18n10> + <42*b18n11> + <42*b18n12> + <42*b18n13> + <42*b18n14> + <42*b18n15> + <42*b18n16> + <42*b18n17> + <42*b18n18> + <42*b19n01> + <42*b19n02> + <42*b19n03> + <42*b19n04> + <42*b19n05> + <42*b19n06> + <42*b19n07> + <42*b19n08> + <42*b19n09> + <42*b19n10> + <42*b19n11> + <42*b19n12> + <42*b19n13> + <42*b19n14> + <42*b19n15> + <42*b19n16> + <42*b19n17> + <42*b19n18> + <42*b20n01> + <42*b20n02> + <42*b20n03> + <42*b20n04> + <42*b20n05> + <42*b20n06> + <42*b20n07> + <42*b20n08> + <42*b20n09> + <42*b20n10> + <42*b20n11> + <42*b20n12> + <42*b20n13> + <42*b20n14> + <42*b20n15> + <42*b20n16> + <42*b20n17> + <42*b20n18> + <42*b21n01> + <42*b21n02> + <42*b21n03> + <42*b21n04> + <42*b21n05> + <42*b21n06> + <42*b21n07> + <42*b21n08> + <42*b21n09> + <42*b21n10> + <42*b21n11> + <42*b21n12> + <42*b21n13> + <42*b21n14> + <42*b21n15> + <42*b21n16> + <42*b21n17> + <42*b21n18> + <42*b22n01> + <42*b22n02> + <42*b22n03> + <42*b22n04> + <42*b22n05> + <42*b22n06> + <42*b22n07> + <42*b22n08> + <42*b22n09> + <42*b22n10> + <42*b22n11> + <42*b22n12> + <42*b22n13> + <42*b22n14> + <42*b22n15> + <42*b22n16> + <42*b22n17> + <42*b22n18> + <42*b23n01> + <42*b23n02> + <42*b23n03> + <42*b23n04> + <42*b23n05> + <42*b23n06> + <42*b23n07> + <42*b23n08> + <42*b23n09> + <42*b23n10> + <42*b23n11> + <42*b23n12> + <42*b23n13> was used as the home directory. was used as the working directory. -Started at Fri Mar 16 11:29:16 2018 -Terminated at Fri Mar 16 11:29:59 2018 -Results reported at Fri Mar 16 11:29:59 2018 +Started at Fri Mar 16 12:45:46 2018 +Terminated at Fri Mar 16 12:56:00 2018 +Results reported at Fri Mar 16 12:56:00 2018 The output (if any) is above this job summary.